diff --git a/backend/src/main/java/com/bakdata/conquery/mode/StorageListener.java b/backend/src/main/java/com/bakdata/conquery/mode/StorageListener.java index fc239f8eac..ae236740ec 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/StorageListener.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/StorageListener.java @@ -6,22 +6,32 @@ import com.bakdata.conquery.models.identifiable.ids.specific.ConceptId; import com.bakdata.conquery.models.identifiable.ids.specific.SecondaryIdDescriptionId; import com.bakdata.conquery.models.identifiable.ids.specific.TableId; +import com.bakdata.conquery.models.jobs.JobManager; +import com.bakdata.conquery.models.worker.DatasetRegistry; +import com.bakdata.conquery.models.worker.DistributedNamespace; +import com.bakdata.conquery.models.worker.Namespace; +import lombok.Data; +import lombok.RequiredArgsConstructor; /** * Listener for updates of stored entities in ConQuery. */ -public interface StorageListener { +@Data +public abstract class StorageListener{ - void onAddSecondaryId(SecondaryIdDescription secondaryId); + private final JobManager jobManager; + private final DatasetRegistry datasetRegistry; - void onDeleteSecondaryId(SecondaryIdDescriptionId description); + public abstract void onAddSecondaryId(SecondaryIdDescription secondaryId); - void onAddTable(Table table); + public abstract void onDeleteSecondaryId(SecondaryIdDescriptionId description); - void onRemoveTable(TableId table); + public abstract void onAddTable(Table table); - void onAddConcept(Concept concept); + public abstract void onRemoveTable(TableId table); - void onDeleteConcept(ConceptId concept); + public abstract void onAddConcept(Concept concept); + + public abstract void onDeleteConcept(ConceptId concept); } diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterStorageListener.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterStorageListener.java index 79dfe49306..9516d7a266 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterStorageListener.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterStorageListener.java @@ -18,49 +18,48 @@ import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.DistributedNamespace; import com.bakdata.conquery.models.worker.WorkerHandler; -import lombok.AllArgsConstructor; /** * Propagates changes of stored entities to relevant ConQuery shards in the cluster. */ -@AllArgsConstructor -public -class ClusterStorageListener implements StorageListener { +public class ClusterStorageListener extends StorageListener { - private final JobManager jobManager; - private final DatasetRegistry datasetRegistry; + + public ClusterStorageListener(JobManager jobManager, DatasetRegistry datasetRegistry) { + super(jobManager, datasetRegistry); + } @Override public void onAddSecondaryId(SecondaryIdDescription secondaryId) { - datasetRegistry.get(secondaryId.getDataset()).getWorkerHandler().sendToAll(new UpdateSecondaryId(secondaryId)); + getDatasetRegistry().get(secondaryId.getDataset()).getWorkerHandler().sendToAll(new UpdateSecondaryId(secondaryId)); } @Override public void onDeleteSecondaryId(SecondaryIdDescriptionId secondaryId) { - datasetRegistry.get(secondaryId.getDataset()).getWorkerHandler().sendToAll(new RemoveSecondaryId(secondaryId)); + getDatasetRegistry().get(secondaryId.getDataset()).getWorkerHandler().sendToAll(new RemoveSecondaryId(secondaryId)); } @Override public void onAddTable(Table table) { - datasetRegistry.get(table.getDataset()).getWorkerHandler().sendToAll(new UpdateTable(table)); + getDatasetRegistry().get(table.getDataset()).getWorkerHandler().sendToAll(new UpdateTable(table)); } @Override public void onRemoveTable(TableId table) { - datasetRegistry.get(table.getDataset()).getWorkerHandler().sendToAll(new RemoveTable(table)); + getDatasetRegistry().get(table.getDataset()).getWorkerHandler().sendToAll(new RemoveTable(table)); } @Override public void onAddConcept(Concept concept) { - WorkerHandler handler = datasetRegistry.get(concept.getDataset()).getWorkerHandler(); + WorkerHandler handler = getDatasetRegistry().get(concept.getDataset()).getWorkerHandler(); SimpleJob simpleJob = new SimpleJob(String.format("sendToAll : Add %s ", concept.getId()), () -> handler.sendToAll(new UpdateConcept(concept))); - jobManager.addSlowJob(simpleJob); + getJobManager().addSlowJob(simpleJob); } @Override public void onDeleteConcept(ConceptId concept) { - WorkerHandler handler = datasetRegistry.get(concept.getDataset()).getWorkerHandler(); + WorkerHandler handler = getDatasetRegistry().get(concept.getDataset()).getWorkerHandler(); SimpleJob simpleJob = new SimpleJob("sendToAll: remove " + concept, () -> handler.sendToAll(new RemoveConcept(concept))); - jobManager.addSlowJob(simpleJob); + getJobManager().addSlowJob(simpleJob); } } diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java index a8d9e2e88d..75db5e7900 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java @@ -11,6 +11,7 @@ import com.bakdata.conquery.mode.NamespaceHandler; import com.bakdata.conquery.mode.cluster.InternalMapperFactory; import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.models.jobs.JobManager; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.LocalNamespace; import com.bakdata.conquery.models.worker.ShardNodeInformation; @@ -33,19 +34,20 @@ public LocalManagerProvider(SqlDialectFactory dialectFactory) { public DelegateManager provideManager(ConqueryConfig config, Environment environment) { + final JobManager jobManager = ManagerProvider.newJobManager(config); + final MetaStorage storage = new MetaStorage(config.getStorage()); final InternalMapperFactory internalMapperFactory = new InternalMapperFactory(config, environment.getValidator()); final NamespaceHandler namespaceHandler = new LocalNamespaceHandler(config, internalMapperFactory, dialectFactory); final DatasetRegistry datasetRegistry = ManagerProvider.createDatasetRegistry(namespaceHandler, config, internalMapperFactory); - return new DelegateManager<>( config, environment, datasetRegistry, storage, new FailingImportHandler(), - new LocalStorageListener(), + new LocalStorageListener(jobManager, datasetRegistry), EMPTY_NODE_PROVIDER, List.of(), internalMapperFactory, diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java index 86e349e5bb..7a3981a7a5 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java @@ -16,6 +16,7 @@ import com.bakdata.conquery.sql.DSLContextWrapper; import com.bakdata.conquery.sql.DslContextFactory; import com.bakdata.conquery.sql.conquery.SqlExecutionManager; +import com.bakdata.conquery.sql.conquery.SqlMatchingStats; import com.bakdata.conquery.sql.conversion.NodeConversions; import com.bakdata.conquery.sql.conversion.SqlConverter; import com.bakdata.conquery.sql.conversion.dialect.SqlDialect; @@ -72,7 +73,8 @@ public LocalNamespace createNamespace(NamespaceStorage namespaceStorage, MetaSto sqlStorageHandler, namespaceData.jobManager(), namespaceData.filterSearch(), - sqlEntityResolver + sqlEntityResolver, + databaseConfig ); } diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalStorageListener.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalStorageListener.java index 8027cabd3e..b4a2ee720d 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalStorageListener.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalStorageListener.java @@ -4,13 +4,22 @@ import com.bakdata.conquery.models.datasets.SecondaryIdDescription; import com.bakdata.conquery.models.datasets.Table; import com.bakdata.conquery.models.datasets.concepts.Concept; +import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; import com.bakdata.conquery.models.identifiable.ids.specific.ConceptId; import com.bakdata.conquery.models.identifiable.ids.specific.SecondaryIdDescriptionId; import com.bakdata.conquery.models.identifiable.ids.specific.TableId; -import lombok.Data; +import com.bakdata.conquery.models.jobs.JobManager; +import com.bakdata.conquery.models.worker.DatasetRegistry; +import com.bakdata.conquery.models.worker.LocalNamespace; -@Data -public class LocalStorageListener implements StorageListener { +public class LocalStorageListener extends StorageListener { + + + public LocalStorageListener( + JobManager jobManager, + DatasetRegistry datasetRegistry) { + super(jobManager, datasetRegistry); + } @Override public void onAddSecondaryId(SecondaryIdDescription secondaryId) { @@ -31,9 +40,13 @@ public void onRemoveTable(TableId table) { @Override public void onAddConcept(Concept concept) { + LocalNamespace namespace = getDatasetRegistry().get(concept.getDataset()); + namespace.getMatchingStats().createConceptIdJoinTable((TreeConcept) concept); } @Override public void onDeleteConcept(ConceptId concept) { + LocalNamespace namespace = getDatasetRegistry().get(concept.getDataset()); + namespace.getMatchingStats().deleteConceptIdJoinTable(concept); } } diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/UpdateMatchingStatsSqlJob.java b/backend/src/main/java/com/bakdata/conquery/mode/local/UpdateMatchingStatsSqlJob.java new file mode 100644 index 0000000000..cbd32c6047 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/UpdateMatchingStatsSqlJob.java @@ -0,0 +1,76 @@ +package com.bakdata.conquery.mode.local; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import com.bakdata.conquery.models.datasets.Dataset; +import com.bakdata.conquery.models.datasets.concepts.Concept; +import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; +import com.bakdata.conquery.models.jobs.Job; +import com.bakdata.conquery.sql.conquery.SqlMatchingStats; +import com.google.common.base.Stopwatch; +import lombok.Data; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.builder.ToStringExclude; + +@Slf4j +@Data +public class UpdateMatchingStatsSqlJob extends Job { + + @ToString.Exclude + private final List> concepts; + private final Dataset dataset; + + @ToString.Exclude + private final SqlMatchingStats matchingStats; + + + @Override + public void execute() throws Exception { + + log.info("BEGIN collecting SQL matching stats for {}", dataset); + + Stopwatch stopwatch = Stopwatch.createStarted(); + + ExecutorService executorService = Executors.newSingleThreadExecutor(); + + List> jobs = new ArrayList<>(); + + + for (Concept concept : concepts) { + if (!(concept instanceof TreeConcept)) { + continue; + } + jobs.add(matchingStats.collectMatchingStatsForConcept((TreeConcept) concept, executorService).toCompletableFuture()); + } + + CompletableFuture all = CompletableFuture.allOf(jobs.toArray(CompletableFuture[]::new)); + while (!all.isDone()) { + if (isCancelled()) { + all.cancel(true); + log.debug("CANCELLED update matching stats for {}", getDataset(), all.exceptionNow()); + return; + } + + all.get(5, TimeUnit.SECONDS); + log.trace("WAITING for matching stats to finish {}", getDataset()); + + if (all.isCompletedExceptionally()) { + log.error("FAILED update matching stats for {}", getDataset(), all.exceptionNow()); + return; + } + } + + log.debug("DONE collecting SQL matching stats for {} within {}", dataset, stopwatch); + } + + @Override + public String getLabel() { + return "Collect matching stats for %s (%s concepts)".formatted(dataset.getName(), concepts.size()); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/MatchingStats.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/MatchingStats.java index 293d845f7d..24cf785758 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/MatchingStats.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/MatchingStats.java @@ -6,10 +6,6 @@ import java.util.Set; import com.bakdata.conquery.models.common.daterange.CDateRange; -import com.bakdata.conquery.models.datasets.Column; -import com.bakdata.conquery.models.datasets.Table; -import com.bakdata.conquery.models.events.Bucket; -import com.bakdata.conquery.models.identifiable.ids.specific.WorkerId; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.AllArgsConstructor; import lombok.Data; @@ -19,77 +15,64 @@ @Getter @Setter +@NoArgsConstructor public class MatchingStats { - private Map entries = new HashMap<>(); - @JsonIgnore - private transient CDateRange span; - - @JsonIgnore - private transient long numberOfEvents = -1L; - - @JsonIgnore - private transient long numberOfEntities = -1L; - - public long countEvents() { - if (numberOfEvents == -1L) { - synchronized (this) { - if (numberOfEvents == -1L) { - numberOfEvents = entries.values().stream().mapToLong(Entry::getNumberOfEvents).sum(); - } - } - } - return numberOfEvents; - } - - - public long countEntities() { - if (numberOfEntities == -1L) { - synchronized (this) { - if (numberOfEntities == -1L) { - numberOfEntities = entries.values().stream().mapToLong(Entry::getNumberOfEntities).sum(); - } - } - } - return numberOfEntities; - } - - public CDateRange spanEvents() { - if (span == null) { - synchronized (this) { - if (span == null) { - span = entries.values().stream().map(Entry::getSpan).reduce(CDateRange.all(), CDateRange::spanClosed); - } - } - } - return span; - - } - - public void putEntry(WorkerId source, Entry entry) { - synchronized (this) { - entries.put(source, entry); - span = null; - numberOfEntities = -1L; - numberOfEvents = -1L; - } - } - - @Data - @NoArgsConstructor - @AllArgsConstructor - public static class Entry { - private long numberOfEvents; - - @JsonIgnore - private final Set foundEntities = new HashSet<>(); - private long numberOfEntities; + private Map entries = new HashMap<>(); + @JsonIgnore + private CDateRange span; + + @JsonIgnore + private long numberOfEvents = -1L; + + @JsonIgnore + private long numberOfEntities = -1L; + + public synchronized long countEvents() { + if (numberOfEvents == -1L) { + numberOfEvents = entries.values().stream().mapToLong(Entry::getNumberOfEvents).sum(); + } + return numberOfEvents; + } + + + public synchronized long countEntities() { + if (numberOfEntities == -1L) { + numberOfEntities = entries.values().stream().mapToLong(Entry::getNumberOfEntities).sum(); + } + return numberOfEntities; + } + + public synchronized CDateRange spanEvents() { + if (span == null) { + span = entries.values().stream().map(Entry::getSpan).reduce(CDateRange.all(), CDateRange::spanClosed); + } + return span; + + } + + public synchronized void putEntry(String source, Entry entry) { + entries.put(source, entry); + span = null; + numberOfEntities = -1L; + numberOfEvents = -1L; + } + + + @Data + @NoArgsConstructor + @AllArgsConstructor + public static class Entry { + @JsonIgnore + private final Set foundEntities = new HashSet<>(); + private long numberOfEvents; + private long numberOfEntities; private int minDate = Integer.MAX_VALUE; private int maxDate = Integer.MIN_VALUE; @JsonIgnore public CDateRange getSpan() { - if(minDate == Integer.MAX_VALUE && maxDate == Integer.MIN_VALUE) { + if (minDate == Integer.MAX_VALUE && maxDate == Integer.MIN_VALUE) { return null; } @@ -99,32 +82,24 @@ public CDateRange getSpan() { ); } - public void addEvent(Table table, Bucket bucket, int event, String entityForEvent) { - numberOfEvents++; - if (foundEntities.add(entityForEvent)) { - numberOfEntities++; - } - - for (Column c : table.getColumns()) { - if (!c.getType().isDateCompatible()) { - continue; - } - - if (!bucket.has(event, c)) { - continue; - } - - final CDateRange time = bucket.getAsDateRange(event, c); - - if (time.hasUpperBound()){ - maxDate = Math.max(time.getMaxValue(), maxDate); - } - - if (time.hasLowerBound()){ - minDate = Math.min(time.getMinValue(), minDate); - } - } - } - } + public void addEvents(String entityForEvent, int events, CDateRange time) { + numberOfEvents += events; + if (foundEntities.add(entityForEvent)) { + numberOfEntities++; + } + + if (time == null) { + return; + } + + if (time.hasUpperBound()) { + maxDate = Math.max(time.getMaxValue(), maxDate); + } + + if (time.hasLowerBound()) { + minDate = Math.min(time.getMinValue(), minDate); + } + } + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/AndCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/AndCondition.java index df9b2fce32..0b248734af 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/AndCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/AndCondition.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.models.datasets.concepts.conditions; +import java.util.Collections; import java.util.List; import java.util.Map; import jakarta.validation.Valid; @@ -52,4 +53,18 @@ public WhereCondition convertToSqlCondition(CTConditionContext context) { () -> new IllegalStateException("At least one condition is required to convert %s to a SQL condition.".formatted(getClass())) ); } + + @Override + public Expression buildExpression(CTConditionContext context, ConceptElement id) { + List expressions = conditions.stream().map(cond -> cond.buildExpression(context, id)) + .toList(); + + Expression out = new Expression(id, Collections.emptyMap()); + + for (Expression expression : expressions) { + out = out.and(expression); + } + + return out; + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/CTCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/CTCondition.java index f334c24dc6..4bcc475389 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/CTCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/CTCondition.java @@ -1,6 +1,9 @@ package com.bakdata.conquery.models.datasets.concepts.conditions; +import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import com.bakdata.conquery.io.cps.CPSBase; import com.bakdata.conquery.models.datasets.concepts.ConceptElement; @@ -9,19 +12,70 @@ import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; import com.bakdata.conquery.util.CalculatedValue; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.google.common.collect.Sets; +import org.jooq.Field; +import org.jooq.Param; /** * A general condition that serves as a guard for concept tree nodes. */ -@JsonTypeInfo(use=JsonTypeInfo.Id.CUSTOM, property="type") +@JsonTypeInfo(use = JsonTypeInfo.Id.CUSTOM, property = "type") @CPSBase public interface CTCondition { default void init(ConceptElement node) throws ConceptConfigurationException { } - + boolean matches(String value, CalculatedValue> rowMap) throws ConceptConfigurationException; + //TODO implement using join-table WhereCondition convertToSqlCondition(CTConditionContext context); + Expression buildExpression(CTConditionContext context, ConceptElement id); + + + /** + * @param conceptElement The conceptElement being defined by the conditions + * @param conditions The conditions defining the conceptElement. Fields are assumed to be and-ed, multiple entries in a field are or-ed. + * So a definition of `{"a": [1], "b": [1,2]}` emits the rows [{a=1 AND b=1}, {a=1 AND b=2}]. + * + */ + //TODO better name + record Expression(ConceptElement conceptElement, Map, Set>> conditions) { + public Expression and(Expression other) { + if (other == null) { + return this; + } + + Set> fields = new HashSet<>(); + fields.addAll(other.conditions.keySet()); + fields.addAll(conditions.keySet()); + + Map, Set>> combined = new HashMap<>(conditions().size() + other.conditions().size()); + + // AND combine fields, if both are present. + for (Field field : fields) { + Set> otherParams = other.conditions.get(field); + Set> myParams = conditions.get(field); + + Set> fieldParams; + + if (otherParams == null || otherParams.isEmpty()) { + fieldParams = myParams; + } + else if (myParams == null || myParams.isEmpty()) { + fieldParams = otherParams; + } + else { + fieldParams = Sets.intersection(otherParams, myParams); + } + + combined.put(field, fieldParams); + } + + return new Expression(conceptElement(), combined); + } + } + + } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/ColumnEqualCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/ColumnEqualCondition.java index 1b98784e95..9d65b1a9ad 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/ColumnEqualCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/ColumnEqualCondition.java @@ -1,16 +1,22 @@ package com.bakdata.conquery.models.datasets.concepts.conditions; +import static org.jooq.impl.DSL.*; +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.SQLDataType.VARCHAR; + import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; +import jakarta.validation.constraints.NotEmpty; import com.bakdata.conquery.io.cps.CPSType; +import com.bakdata.conquery.models.datasets.concepts.ConceptElement; import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext; import com.bakdata.conquery.sql.conversion.model.filter.MultiSelectCondition; import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; import com.bakdata.conquery.util.CalculatedValue; import com.bakdata.conquery.util.CollectionsUtil; import com.fasterxml.jackson.annotation.JsonCreator; -import jakarta.validation.constraints.NotEmpty; import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Getter; @@ -46,7 +52,13 @@ public boolean matches(String value, CalculatedValue> rowMap @Override public WhereCondition convertToSqlCondition(CTConditionContext context) { - Field field = DSL.field(DSL.name(context.getConnectorTable().getName(), column), String.class); + Field field = field(name(column), String.class); return new MultiSelectCondition(field, values.toArray(String[]::new), context.getFunctionProvider()); } + + @Override + public Expression buildExpression(CTConditionContext context, ConceptElement id) { + return new Expression(id, Map.of(field(name(getColumn()), VARCHAR(32)).as("%s_equal".formatted(column)), values.stream().map(DSL::val).collect(Collectors.toSet()))); + + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/EqualCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/EqualCondition.java index 80e3e104a6..5d5a1f2e3a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/EqualCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/EqualCondition.java @@ -1,31 +1,36 @@ package com.bakdata.conquery.models.datasets.concepts.conditions; +import static org.jooq.impl.DSL.field; + import java.util.Map; import java.util.Set; - +import java.util.stream.Collectors; import jakarta.validation.constraints.NotEmpty; import com.bakdata.conquery.io.cps.CPSType; +import com.bakdata.conquery.models.datasets.concepts.ConceptElement; import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext; import com.bakdata.conquery.sql.conversion.model.filter.MultiSelectCondition; import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; import com.bakdata.conquery.util.CalculatedValue; import com.bakdata.conquery.util.CollectionsUtil; import com.fasterxml.jackson.annotation.JsonCreator; +import com.google.common.base.Preconditions; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.Setter; -import org.jooq.Field; import org.jooq.impl.DSL; /** * This condition requires each value to be exactly as given in the list. */ -@CPSType(id="EQUAL", base=CTCondition.class) +@CPSType(id = "EQUAL", base = CTCondition.class) @AllArgsConstructor public class EqualCondition implements CTCondition { - @Setter @Getter @NotEmpty + @Setter + @Getter + @NotEmpty private Set values; @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) @@ -40,7 +45,11 @@ public boolean matches(String value, CalculatedValue> rowMap @Override public WhereCondition convertToSqlCondition(CTConditionContext context) { - Field field = DSL.field(DSL.name(context.getConnectorTable().getName(), context.getConnectorColumn().getName()), String.class); - return new MultiSelectCondition(field, values.toArray(String[]::new), context.getFunctionProvider()); + return new MultiSelectCondition(context.getConnectorColumn(), values.toArray(String[]::new), context.getFunctionProvider()); + } + + @Override + public Expression buildExpression(CTConditionContext context, ConceptElement id) { + return new Expression(id, Map.of(context.getConnectorColumn(), values.stream().map(DSL::val).collect(Collectors.toSet()))); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/GroovyCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/GroovyCondition.java index a7bd4f6a97..42a8b297d6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/GroovyCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/GroovyCondition.java @@ -26,6 +26,7 @@ */ @Slf4j @CPSType(id = "GROOVY", base = CTCondition.class) +@Deprecated public class GroovyCondition implements CTCondition { public static final String[] AUTO_IMPORTS = Stream.of( @@ -117,4 +118,9 @@ public Object getProperty(String property) { } } } + + @Override + public Expression buildExpression(CTConditionContext context, ConceptElement id) { + throw new IllegalStateException("Not implemented"); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/IsEmptyCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/IsEmptyCondition.java new file mode 100644 index 0000000000..0012166573 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/IsEmptyCondition.java @@ -0,0 +1,47 @@ +package com.bakdata.conquery.models.datasets.concepts.conditions; + +import static org.jooq.impl.DSL.*; +import static org.jooq.impl.SQLDataType.BOOLEAN; + +import java.util.Map; +import java.util.Set; + +import com.bakdata.conquery.io.cps.CPSType; +import com.bakdata.conquery.models.datasets.concepts.ConceptElement; +import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext; +import com.bakdata.conquery.sql.conversion.model.filter.ConditionWrappingWhereCondition; +import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; +import com.bakdata.conquery.util.CalculatedValue; +import lombok.Getter; +import lombok.NonNull; +import lombok.Setter; +import org.jooq.Condition; +import org.jooq.impl.DSL; + +/** + * This condition requires that the selected Column has a value. + */ +@CPSType(id = "NOT_PRESENT", base = CTCondition.class) +public class IsEmptyCondition implements CTCondition { + + @Getter + @Setter + @NonNull + private String column; + + @Override + public boolean matches(String value, CalculatedValue> rowMap) { + return rowMap.getValue().containsKey(column); + } + + @Override + public WhereCondition convertToSqlCondition(CTConditionContext context) { + Condition condition = field(name(column)).isNull(); + return new ConditionWrappingWhereCondition(condition); + } + + @Override + public Expression buildExpression(CTConditionContext context, ConceptElement id) { + return new Expression(id, Map.of(field(name(column), BOOLEAN).isNull().as("%s_is_empty".formatted(column)), Set.of(val(true)))); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/IsPresentCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/IsPresentCondition.java index 783e98e7d0..7093f602e5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/IsPresentCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/IsPresentCondition.java @@ -1,8 +1,12 @@ package com.bakdata.conquery.models.datasets.concepts.conditions; +import static org.jooq.impl.DSL.*; + import java.util.Map; +import java.util.Set; import com.bakdata.conquery.io.cps.CPSType; +import com.bakdata.conquery.models.datasets.concepts.ConceptElement; import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext; import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; import com.bakdata.conquery.sql.conversion.model.filter.ConditionWrappingWhereCondition; @@ -30,7 +34,12 @@ public boolean matches(String value, CalculatedValue> rowMap @Override public WhereCondition convertToSqlCondition(CTConditionContext context) { - Condition condition = DSL.field(DSL.name(context.getConnectorTable().getName(), column)).isNotNull(); + Condition condition = field(name(column)).isNotNull(); return new ConditionWrappingWhereCondition(condition); } + + @Override + public Expression buildExpression(CTConditionContext context, ConceptElement id) { + return new Expression(id, Map.of(field(name(column)).isNull().as("%s_is_empty".formatted(column)), Set.of(val(false)))); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/NotCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/NotCondition.java index 3d5e9ff9e5..effaa12710 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/NotCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/NotCondition.java @@ -16,6 +16,7 @@ * This condition matches if its child does not. */ @CPSType(id="NOT", base=CTCondition.class) +@Deprecated public class NotCondition implements CTCondition { @Setter @Getter @Valid @@ -36,4 +37,9 @@ public WhereCondition convertToSqlCondition(CTConditionContext context) { WhereCondition whereCondition = condition.convertToSqlCondition(context); return whereCondition.negate(); } + + @Override + public Expression buildExpression(CTConditionContext context, ConceptElement id) { + throw new IllegalStateException("Not implemented"); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/OrCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/OrCondition.java index b5b0d8b2bb..f078e4fcde 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/OrCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/OrCondition.java @@ -17,6 +17,7 @@ /** * This condition connects multiple conditions with an or. */ +@Deprecated @CPSType(id = "OR", base = CTCondition.class) public class OrCondition implements CTCondition { @@ -52,4 +53,9 @@ public WhereCondition convertToSqlCondition(CTConditionContext context) { () -> new IllegalStateException("At least one condition is required to convert %s to a SQL condition.".formatted(getClass())) ); } + + @Override + public Expression buildExpression(CTConditionContext context, ConceptElement id) { + throw new IllegalStateException("Not implemented"); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/PrefixCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/PrefixCondition.java index 5f90cd47b5..8d4ed89939 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/PrefixCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/PrefixCondition.java @@ -1,10 +1,13 @@ package com.bakdata.conquery.models.datasets.concepts.conditions; +import static org.jooq.impl.DSL.field; + import java.util.Arrays; import java.util.Map; import java.util.stream.Collectors; import com.bakdata.conquery.io.cps.CPSType; +import com.bakdata.conquery.models.datasets.concepts.ConceptElement; import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext; import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; import com.bakdata.conquery.sql.conversion.model.filter.ConditionWrappingWhereCondition; @@ -14,14 +17,13 @@ import lombok.Setter; import lombok.ToString; import org.jooq.Condition; -import org.jooq.Field; -import org.jooq.impl.DSL; /** * This condition requires each value to start with one of the given values. */ @CPSType(id = "PREFIX_LIST", base = CTCondition.class) @ToString +@Deprecated public class PrefixCondition implements CTCondition { @Setter @@ -41,9 +43,14 @@ public boolean matches(String value, CalculatedValue> rowMap @Override public WhereCondition convertToSqlCondition(CTConditionContext context) { - Field field = DSL.field(DSL.name(context.getConnectorTable().getName(), context.getConnectorColumn().getName()), String.class); String pattern = Arrays.stream(prefixes).collect(Collectors.joining("|", "", context.getFunctionProvider().getAnyCharRegex())); - Condition condition = context.getFunctionProvider().likeRegex(field, pattern); + Condition condition = context.getFunctionProvider().likeRegex(context.getConnectorColumn(), pattern); return new ConditionWrappingWhereCondition(condition); } + + @Override + public Expression buildExpression(CTConditionContext context, ConceptElement id) { + // Implementation is technically possible but extremely slow and PREFIX has caused issues historically + throw new IllegalStateException("Not implemented"); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/PrefixRangeCondition.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/PrefixRangeCondition.java index 66219f1366..8d093a32c3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/PrefixRangeCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/PrefixRangeCondition.java @@ -1,9 +1,12 @@ package com.bakdata.conquery.models.datasets.concepts.conditions; +import static org.jooq.impl.DSL.field; + import java.util.Map; import jakarta.validation.constraints.NotEmpty; import com.bakdata.conquery.io.cps.CPSType; +import com.bakdata.conquery.models.datasets.concepts.ConceptElement; import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; @@ -15,12 +18,12 @@ import lombok.Setter; import org.jooq.Condition; import org.jooq.Field; -import org.jooq.impl.DSL; /** * This condition requires each value to start with a prefix between the two given values */ @CPSType(id = "PREFIX_RANGE", base = CTCondition.class) +@Deprecated public class PrefixRangeCondition implements CTCondition { private static final String ANY_CHAR_REGEX = ".*"; @@ -54,7 +57,7 @@ public boolean matches(String value, CalculatedValue> rowMap @Override public WhereCondition convertToSqlCondition(CTConditionContext context) { - Field field = DSL.field(DSL.name(context.getConnectorTable().getName(), context.getConnectorColumn().getName()), String.class); + Field field = context.getConnectorColumn(); String pattern = buildSqlRegexPattern(context.getFunctionProvider()); Condition regexCondition = context.getFunctionProvider().likeRegex(field, pattern); return new ConditionWrappingWhereCondition(regexCondition); @@ -76,4 +79,9 @@ private String buildSqlRegexPattern(SqlFunctionProvider functionProvider) { } return builder.append(functionProvider.getAnyCharRegex()).toString(); } + + @Override + public Expression buildExpression(CTConditionContext context, ConceptElement id) { + throw new IllegalStateException("Not implemented"); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/UpdateElementMatchingStats.java b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/UpdateElementMatchingStats.java index 9d5821b198..ffd7afb675 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/UpdateElementMatchingStats.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/UpdateElementMatchingStats.java @@ -33,6 +33,8 @@ public class UpdateElementMatchingStats extends NamespaceMessage { @Override public void react(DistributedNamespace context) throws Exception { + String sourceString = source.toString(); + // We collect the concepts outside the loop to update the storage afterward Map> conceptsToUpdate = new HashMap<>(); @@ -56,7 +58,7 @@ public void react(DistributedNamespace context) throws Exception { matchingStats = new MatchingStats(); target.setMatchingStats(matchingStats); } - matchingStats.putEntry(source, value); + matchingStats.putEntry(sourceString, value); } catch (Exception e) { log.error("Failed to set matching stats for '{}' (enable TRACE for exception)", entry.getKey(), (Exception) (log.isTraceEnabled() ? e : null)); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/UpdateMatchingStatsMessage.java b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/UpdateMatchingStatsMessage.java index 3e9407b720..7a6927360a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/UpdateMatchingStatsMessage.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/UpdateMatchingStatsMessage.java @@ -1,7 +1,9 @@ package com.bakdata.conquery.models.messages.namespaces.specific; +import java.util.Arrays; import java.util.Collection; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; @@ -11,6 +13,8 @@ import java.util.stream.Stream; import com.bakdata.conquery.io.cps.CPSType; +import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.Table; import com.bakdata.conquery.models.datasets.concepts.Concept; import com.bakdata.conquery.models.datasets.concepts.ConceptElement; @@ -33,6 +37,7 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.mina.core.future.WriteFuture; +import org.jetbrains.annotations.Nullable; /** * For each {@link com.bakdata.conquery.models.query.queryplan.specific.ConceptNode} calculate the number of matching events and the span of date-ranges. @@ -45,6 +50,45 @@ public class UpdateMatchingStatsMessage extends WorkerMessage { @Getter private final Collection concepts; + @Nullable + private static CDateRange spannedValidityDates(Bucket bucket, int event, Iterable dateColumns) { + int maxDate = Integer.MIN_VALUE; + int minDate = Integer.MAX_VALUE; + + for (Column c : dateColumns) { + + if (!bucket.has(event, c)) { + continue; + } + + final CDateRange time = bucket.getAsDateRange(event, c); + + if (time.hasUpperBound()) { + maxDate = Math.max(time.getMaxValue(), maxDate); + } + + if (time.hasLowerBound()) { + minDate = Math.min(time.getMinValue(), minDate); + } + } + + final CDateRange span; + + if (minDate == Integer.MAX_VALUE && maxDate == Integer.MIN_VALUE) { + span = null; + } + else if (minDate == Integer.MAX_VALUE) { + span = CDateRange.atMost(maxDate); + } + else if (maxDate == Integer.MIN_VALUE) { + span = CDateRange.atLeast(minDate); + } + else { + span = CDateRange.of(minDate, maxDate); + } + return span; + } + @Override public void react(Worker worker) throws Exception { @@ -144,6 +188,7 @@ private static Map, MatchingStats.Entry> calculateConceptMat CBlock cBlock = cBlockId.resolve(); final Bucket bucket = cBlock.getBucket().resolve(); final Table table = bucket.getTable().resolve(); + final List dateColumns = Arrays.stream(table.getColumns()).filter(c -> c.getType().isDateCompatible()).toList(); for (String entity : bucket.entities()) { @@ -152,10 +197,12 @@ private static Map, MatchingStats.Entry> calculateConceptMat for (int event = bucket.getEntityStart(entity); event < entityEnd; event++) { final int[] localIds = cBlock.getPathToMostSpecificChild(event); + final CDateRange span = spannedValidityDates(bucket, event, dateColumns); if (!(concept instanceof TreeConcept) || localIds == null) { - matchingStats.computeIfAbsent(conceptId, (ignored) -> new MatchingStats.Entry()).addEvent(table, bucket, event, entity); + matchingStats.computeIfAbsent(conceptId, (ignored) -> new MatchingStats.Entry()) + .addEvents(entity, 1, span); continue; } @@ -167,7 +214,7 @@ private static Map, MatchingStats.Entry> calculateConceptMat while (element != null) { matchingStats.computeIfAbsent(element.getId(), (ignored) -> new MatchingStats.Entry()) - .addEvent(table, bucket, event, entity); + .addEvents(entity, 1, span); element = element.getParent(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java b/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java index 78c43ecc5c..a487a865ec 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java @@ -8,10 +8,13 @@ import com.bakdata.conquery.io.storage.NamespaceStorage; import com.bakdata.conquery.mode.local.SqlEntityResolver; import com.bakdata.conquery.mode.local.SqlStorageHandler; +import com.bakdata.conquery.mode.local.UpdateMatchingStatsSqlJob; +import com.bakdata.conquery.models.config.DatabaseConfig; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.jobs.JobManager; import com.bakdata.conquery.models.query.ExecutionManager; import com.bakdata.conquery.sql.DSLContextWrapper; +import com.bakdata.conquery.sql.conquery.SqlMatchingStats; import com.bakdata.conquery.sql.conversion.dialect.SqlDialect; import com.bakdata.conquery.util.search.SearchProcessor; import com.fasterxml.jackson.databind.ObjectMapper; @@ -25,6 +28,8 @@ public class LocalNamespace extends Namespace { private final SqlDialect dialect; private final DSLContextWrapper dslContextWrapper; private final SqlStorageHandler storageHandler; + private final DatabaseConfig databaseConfig; + private final SqlMatchingStats matchingStats; public LocalNamespace( SqlDialect dialect, @@ -35,17 +40,22 @@ public LocalNamespace( SqlStorageHandler storageHandler, JobManager jobManager, SearchProcessor filterSearch, - SqlEntityResolver sqlEntityResolver + SqlEntityResolver sqlEntityResolver, DatabaseConfig databaseConfig ) { super(preprocessMapper, storage, executionManager, jobManager, filterSearch, sqlEntityResolver); this.dslContextWrapper = dslContextWrapper; this.storageHandler = storageHandler; this.dialect = dialect; + this.databaseConfig = databaseConfig; + matchingStats = new SqlMatchingStats(dslContextWrapper.getDslContext(), dialect.getFunctionProvider(), databaseConfig); } + @Override void updateMatchingStats() { - // TODO Build basic statistic on data + getJobManager().addSlowJob( + new UpdateMatchingStatsSqlJob(getStorage().getAllConcepts().toList(), getDataset(), getMatchingStats()) + ); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlMatchingStats.java b/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlMatchingStats.java new file mode 100644 index 0000000000..986215a19e --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlMatchingStats.java @@ -0,0 +1,466 @@ +package com.bakdata.conquery.sql.conquery; + +import static org.jooq.impl.DSL.*; + +import java.sql.Date; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.ExecutorService; +import jakarta.validation.constraints.NotBlank; + +import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.models.config.DatabaseConfig; +import com.bakdata.conquery.models.datasets.Column; +import com.bakdata.conquery.models.datasets.concepts.ConceptElement; +import com.bakdata.conquery.models.datasets.concepts.Connector; +import com.bakdata.conquery.models.datasets.concepts.MatchingStats; +import com.bakdata.conquery.models.datasets.concepts.ValidityDate; +import com.bakdata.conquery.models.datasets.concepts.conditions.CTCondition; +import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeChild; +import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; +import com.bakdata.conquery.models.events.MajorTypeId; +import com.bakdata.conquery.models.identifiable.ids.specific.ConceptElementId; +import com.bakdata.conquery.models.identifiable.ids.specific.ConceptId; +import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext; +import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import com.bakdata.conquery.util.TablePrimaryColumnUtil; +import com.google.common.base.Stopwatch; +import com.google.common.collect.Sets; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; +import org.jooq.CommonTableExpression; +import org.jooq.Condition; +import org.jooq.CreateTableElementListStep; +import org.jooq.Cursor; +import org.jooq.DSLContext; +import org.jooq.Field; +import org.jooq.InsertValuesStepN; +import org.jooq.Name; +import org.jooq.Param; +import org.jooq.Record; +import org.jooq.Record4; +import org.jooq.RowN; +import org.jooq.Select; +import org.jooq.SelectConditionStep; +import org.jooq.SelectJoinStep; +import org.jooq.exception.DataAccessException; + +@Slf4j +@Data +public class SqlMatchingStats { + + private final Field PID_FIELD = field(name("pid"), String.class); + private final Field LB_FIELD = field(name("lower_bound"), Date.class); + private final Field UB_FIELD = field(name("upper_bound"), Date.class); + private final Field CONCEPT_ID_FIELD = field(name("resolved_id"), Integer.class).comment("LocalId of the concept"); + private final Set> NULL_PARAMS = Collections.singleton(inline(null, String.class)); + + private final DSLContext dslContext; + private final SqlFunctionProvider functionProvider; + private final DatabaseConfig dbConfig; + private final int fetchBatchSize = 100; //TODO from dbConfig? + + private static void assignStatsToPath(ConceptElement element, Map, MatchingStats.Entry> matchingStats, String entity, CDateRange span) { + ConceptElementId id = element.getId(); + + while (element != null) { + matchingStats.computeIfAbsent(id, (ignored) -> new MatchingStats.Entry()) + .addEvents(entity, 1, span); + element = element.getParent(); + } + } + + /** + * collect unique fields used/defined in the expressions. + */ + private static List> collectAllFields(List expressions) { + List> fields = expressions.stream() + //TODO determine length of chars, for now we are relying on a fixed length because it's quite cumbersome + .map(expression -> expression.conditions().keySet()) + .flatMap(Collection::stream) + .distinct() + .toList(); + return fields; + } + + private static Select unionSelects(List> connectorTables) { + Select unioned = null; + + for (Select connectorTable : connectorTables) { + if (unioned == null) { + unioned = (Select) connectorTable; + continue; + } + + unioned = unioned.unionAll(connectorTable); + } + + + return unioned; + } + + /** + * Assembles the join table and inserts it into the database. + * @param concept + */ + public void createConceptIdJoinTable(TreeConcept concept) { + CTConditionContext context = CTConditionContext.forJoinTables(functionProvider); + + List expressions = collectAllExpressions(concept, null, context); + + List> allFields = collectAllFields(expressions); + + List rows = expressionsToRows(expressions, allFields); + + Name tableName = idsTableName(concept.getName()); + + // allFields are the statements to extract values from the underlying tables, we use them to generate the field names + List> fields = new ArrayList<>(); + + fields.addAll(allFields); + fields.addFirst(CONCEPT_ID_FIELD); + + createConceptIdsTable(tableName, fields); + insertConceptIdMappings(tableName, fields, rows, dslContext); + } + + @NotNull + private Field[] collectValidityDateFields(Connector connector) { + List> validityDates = new ArrayList<>(); + + for (ValidityDate validityDate : connector.getValidityDates()) { + if (!validityDate.isSingleColumnDaterange()) { + validityDates.add(field(name(validityDate.getStartColumn().getColumn()), Date.class)); + validityDates.add(field(name(validityDate.getEndColumn().getColumn()), Date.class)); + continue; + } + + Column column = validityDate.getColumn().get(); + + if (column.getType() == MajorTypeId.DATE) { + validityDates.add(field(name(column.getName()), Date.class)); + } + else if (column.getType() == MajorTypeId.DATE_RANGE) { + Field rangeField = field(name(column.getName())); + + validityDates.add(functionProvider.lower(rangeField)); + validityDates.add(functionProvider.upper(rangeField)); + } + } + return (Field[]) validityDates.toArray(Field[]::new); + } + + private void assignStats(Map, MatchingStats.Entry> matchingStats) { + for (Map.Entry, MatchingStats.Entry> entry : matchingStats.entrySet()) { + ConceptElementId conceptElementId = entry.getKey(); + + MatchingStats stats = new MatchingStats(); + stats.putEntry("sql", entry.getValue()); + conceptElementId.resolve().setMatchingStats(stats); + } + } + + @NotNull + private Map, MatchingStats.Entry> readStats( + TreeConcept concept, + SelectJoinStep selectJoinStep) { + Map, MatchingStats.Entry> matchingStats = new HashMap<>(); + + Stopwatch stopwatch = Stopwatch.createStarted(); + + log.info("BEGIN fetching matching stats for {}", concept.getId()); + log.trace("{}", selectJoinStep); + + try (Cursor cursor = selectJoinStep.fetchSize(fetchBatchSize).fetchLazy()) { + + for (Record record : cursor) { + + Integer rawId = record.get(CONCEPT_ID_FIELD); + + + ConceptElement resolvedId; + if (rawId == null) { + resolvedId = concept; + } + else { + resolvedId = concept.getElementByLocalId(rawId); + } + + String entity = record.get(PID_FIELD); + Date min = record.get(LB_FIELD); + Date max = record.get(UB_FIELD); + + CDateRange span = CDateRange.of(min != null ? min.toLocalDate() : null, max != null ? max.toLocalDate() : null); + + assignStatsToPath(resolvedId, matchingStats, entity, span); + } + } + + log.debug("DONE fetching matching stats for {} within {}", concept.getId(), stopwatch); + + + return matchingStats; + } + + @NotNull + private Name idsTableName(@NotBlank String name) { + return name("%s_ids".formatted(name)); + } + + private void insertConceptIdMappings(Name tableName, List> fieldNames, List rows, DSLContext dsl) { + log.info("BEGIN inserting {} rows into {}", rows.size(), tableName); + + // We're using batching here because some DBMS don't allow mass inserts. + // There's a chance, we rework this to use a prepared statement with lots of bindings under the hood. But that needs to rework the entire stream of rows. + List> inserts = new ArrayList<>(rows.size()); + + for (RowN row : rows) { + inserts.add(dsl.insertInto(table(tableName)) + .columns(fieldNames) + .values(row)); + } + + dsl.batch(inserts) + .execute(); + + + log.trace("DONE inserting into {}", tableName); + } + + /** + * Drop the table, then recreate it. + * TODO add an index. + */ + private void createConceptIdsTable(Name tableName, List> fields) { + + log.debug("Creating table {} with fields {}", tableName, fields); + + try { + dslContext.dropTable(tableName) + .cascade() + .execute(); + } + catch (DataAccessException exception) { + // Likely it doesn't exist. Some DBMS just don't support drop-IfExists so this is the next best thing :^) + log.trace("Failed to drop table {}", tableName, exception); + } + + CreateTableElementListStep createTable = + dslContext.createTable(tableName) + .columns(fields); + + + createTable.execute(); + + //TODO null values still crash this :'( + // if (!allFields.isEmpty()) { + // String indexName = "%s_index".formatted(tableName.unquotedName().toString()); + // dslContext.dropIndexIfExists(indexName).execute(); + // dslContext.createIndex(indexName) + // .on(table(tableName), allFields.stream().map(Field::sortDefault).toList()) + // .excludeNullKeys() + // .execute(); + // } + } + + private int findMaxIdLength(List expressions) { + return expressions.stream().mapToInt(e -> e.conceptElement().getId().toString().length()).max() + .orElse(0); + } + + public CompletionStage collectMatchingStatsForConcept(TreeConcept concept, ExecutorService executorService) { + + // The transaction implicitly disables autocommit, which we need for using the cursor + return dslContext + .transactionAsync(executorService, cfg -> { + SelectJoinStep matchingStatsStatement = createMatchingStatsStatement(concept); + Map, MatchingStats.Entry> matchingStats = readStats(concept, matchingStatsStatement); + assignStats(matchingStats); + } + ); + + } + + @NotNull + private SelectJoinStep createMatchingStatsStatement(TreeConcept concept) { + + List> connectorTables = new ArrayList<>(); + + Field positiveInfinity = functionProvider.toDateField(functionProvider.getMaxDateExpression()); + Field negativeInfinity = functionProvider.toDateField(functionProvider.getMinDateExpression()); + + for (Connector connector : concept.getConnectors()) { + + CTConditionContext context = CTConditionContext.forConnector(connector, functionProvider); + + Field[] validityDates = collectValidityDateFields(connector); + + SelectConditionStep connectorTable = + dslContext.select( + TablePrimaryColumnUtil.findPrimaryColumn(connector.getResolvedTable(), dbConfig).as(PID_FIELD), + // The infinities are intentionally swapped + least(positiveInfinity, validityDates).as(LB_FIELD), + greatest(negativeInfinity, validityDates).as(UB_FIELD), + CONCEPT_ID_FIELD + ) + .from(table(name(connector.getResolvedTable().getName()))) + .leftJoin(idsTableName(concept.getName())) + .on(getJoinConditions(concept, context)) // join onto the concept-ids table to assign the most specific id. + .where(connector.getCondition() != null ? connector.getCondition().convertToSqlCondition(context).condition() : noCondition()); + + connectorTables.add(connectorTable); + } + + Name ct_name = name("connector_tables"); + CommonTableExpression unioned = ct_name.as(unionSelects(connectorTables)); + + SelectJoinStep> records = + dslContext.with(unioned) + .select( + unioned.field(CONCEPT_ID_FIELD), + PID_FIELD, + // The infinities are intentionally swapped + nullif(unioned.field(LB_FIELD), positiveInfinity).as(LB_FIELD), + nullif(unioned.field(UB_FIELD), negativeInfinity).as(UB_FIELD) + ) + .from(ct_name); + + return records; + } + + public void deleteConceptIdJoinTable(ConceptId concept) { + Name tableName = idsTableName(concept.getName()); + log.debug("Dropping table {}", tableName); + dslContext.dropTableIfExists(tableName) + .cascade() + .execute(); + } + + + /** + * Using the expressions of a concept, build a Condition that descibes the left-join onto the ids table, from any connector-table. + */ + private Condition getJoinConditions(TreeConcept concept, CTConditionContext context) { + List expressions = collectAllExpressions(concept, null, context); + + Collection> allFields = collectAllFields(expressions); + + if (allFields.isEmpty()) { + // TODO this is a HANA-ism: It expects proper expressions in joins + return field(inline(true)).eq(field(inline(true))); + } + + Name idsTable = idsTableName(concept.getName()); + + Condition out = noCondition(); + + for (Field eField : allFields) { + // col_val needs extra handling because it's bound to the connector and not the concept. + if (eField.equals(context.getConnectorColumn())) { + out = out.and(eField.eq(CTConditionContext.COLUMN_VALUE_FIELD)); + continue; + } + + // The conceptElement-tables names are derived from eField so this should work. + out = out.and(eField.eq(field(name(idsTable, eField.getUnqualifiedName())))); + } + + return out; + } + + private List expressionsToRows(List expressions, List> allFields) { + Map>, ConceptElement> byDepth = new HashMap<>(); + + for (CTCondition.Expression expression : expressions) { + ConceptElement elt = expression.conceptElement(); + + List>> rowValues = new ArrayList<>(); + for (Field field : allFields) { + rowValues.add(expression.conditions().getOrDefault(field, NULL_PARAMS)); + } + + Set>> flattened = Sets.cartesianProduct(rowValues); + + // Group by params, find deepest params. This ensures we map to the most-specific element. + for (List> params : flattened) { + byDepth.compute(params, + (__, prior) -> { + if (prior == null || prior.getDepth() < elt.getDepth()) { + return elt; + } + if (prior.getDepth() == elt.getDepth() && !prior.equals(elt)) { + log.warn("Nodes {} and {} are mapped by the same params {}", prior.getId(), elt.getId(), params); + } + return prior; + } + ); + } + } + + List rows = new ArrayList<>(); + + for (Map.Entry>, ConceptElement> entry : byDepth.entrySet()) { + List> params = new ArrayList<>(entry.getKey().size() + 1); + + params.addFirst(val(entry.getValue().getLocalId())); + params.addAll(entry.getKey()); + + rows.add(row(params)); + } + return rows; + } + + /** + * Collect all mappings from values to conceptElement for the entire concept. This means the column-value and the auxiliary columns. + * We use them to construct a table building an injective mapping from values to concept element that can be used for performant joins instead of resolving the concept every time. + */ + private List collectAllExpressions(ConceptElement current, CTCondition.Expression parentExpression, CTConditionContext context) { + + final CTCondition.Expression forCurrent = switch (current) { + case TreeConcept concept -> new CTCondition.Expression(concept, Collections.emptyMap()); + // concept elements implicitly inherit the conditions of its parents + case ConceptTreeChild child -> child.getCondition() + .buildExpression(context, current) + .and(parentExpression); + case null, default -> throw new IllegalStateException(); + }; + + final List out = new ArrayList<>(); + + out.add(forCurrent); + + for (ConceptTreeChild child : current.getChildren()) { + out.addAll(collectAllExpressions(child, forCurrent, context)); + } + + return out; + } + + /** + * recursively build just a single expression + * @param current + * @param context + * + * TODO use this to implement joining in queries + */ + private CTCondition.Expression collectExpressionsForSingleNode(ConceptElement current, CTConditionContext context) { + + if (current instanceof TreeConcept concept) { + return new CTCondition.Expression(concept, Collections.emptyMap()); + } + + CTCondition.Expression parentExpression = collectExpressionsForSingleNode(current.getParent(), context); + CTCondition.Expression currentExpression = ((ConceptTreeChild) current).getCondition().buildExpression(context, current); + + return currentExpression.and(parentExpression); + } + + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java index 7083abdf66..9bdfb7ca9a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java @@ -95,7 +95,7 @@ private QueryStep createRowSelects( } private static SqlIdColumns createIdSelect(Map.Entry entry) { - Field primaryColumn = DSL.val(entry.getKey()).coerce(Object.class).as(SharedAliases.PRIMARY_COLUMN.getAlias()); + Field primaryColumn = DSL.val(entry.getKey()).coerce(String.class).as(SharedAliases.PRIMARY_COLUMN.getAlias()); return new SqlIdColumns(primaryColumn); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java index eaa8dfaa86..0ad894bc98 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.sql.conversion.cqelement; +import static org.jooq.impl.DSL.field; + import com.bakdata.conquery.apiv1.query.CQYes; import com.bakdata.conquery.models.config.ColumnConfig; import com.bakdata.conquery.sql.conversion.NodeConverter; @@ -23,7 +25,7 @@ public Class getConversionClass() { public ConversionContext convert(CQYes cqYes, ConversionContext context) { ColumnConfig primaryColumnConfig = context.getIdColumns().findPrimaryIdColumn(); - Field primaryColumn = DSL.field(DSL.name(primaryColumnConfig.getField())); + Field primaryColumn = field(DSL.name(primaryColumnConfig.getField()), String.class); SqlIdColumns ids = new SqlIdColumns(primaryColumn); Selects selects = Selects.builder().ids(ids).build(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java index 1ce44a066d..063a1130a2 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java @@ -128,7 +128,7 @@ private static QueryStep finishConceptConversion(QueryStep predecessor, CQConcep public static SqlIdColumns convertIds(CQConcept cqConcept, CQTable cqTable, ConversionContext conversionContext) { Table table = cqTable.getConnector().resolve().getResolvedTable(); - Field primaryColumn = TablePrimaryColumnUtil.findPrimaryColumn(table, conversionContext.getConfig()); + Field primaryColumn = TablePrimaryColumnUtil.findPrimaryColumn(table, conversionContext.getConfig()); if (cqConcept.isExcludeFromSecondaryId() || conversionContext.getSecondaryIdDescription() == null @@ -147,7 +147,7 @@ public static SqlIdColumns convertIds(CQConcept cqConcept, CQTable cqTable, Conv ) ); - Field secondaryId = DSL.field(DSL.name(table.getName(), secondaryIdColumn.getName())); + Field secondaryId = DSL.field(DSL.name(table.getName(), secondaryIdColumn.getName()), String.class); return new SqlIdColumns(primaryColumn, secondaryId).withAlias(); } @@ -222,7 +222,9 @@ private static WhereCondition convertConceptElementCondition(ConceptElement c ConceptTreeChild child = (ConceptTreeChild) conceptElement; - WhereCondition childCondition = child.getCondition().convertToSqlCondition(CTConditionContext.create(cqTable.getConnector().resolve(), functionProvider)); + WhereCondition childCondition = child.getCondition().convertToSqlCondition(CTConditionContext.forConnector( + cqTable.getConnector().resolve(), functionProvider + )); WhereCondition parentCondition = convertConceptElementCondition(child.getParent(), cqTable, functionProvider); return parentCondition.and(childCondition); @@ -237,7 +239,7 @@ private static WhereCondition convertConnectorCondition(CQTable cqTable, SqlFunc if (connector.getCondition() == null) { return prerequisites; } - WhereCondition converted = connector.getCondition().convertToSqlCondition(CTConditionContext.create(connector, functionProvider)); + WhereCondition converted = connector.getCondition().convertToSqlCondition(CTConditionContext.forConnector(connector, functionProvider)); return converted.and(prerequisites); } @@ -327,7 +329,8 @@ private CQTableContext createTableContext(TablePath tablePath, CQConcept cqConce List> resolvedConceptElements = cqConcept.getElements().stream().>map(ConceptElementId::resolve).toList(); allSqlFiltersForTable.add(collectConceptConditions(resolvedConceptElements, cqTable, functionProvider, ids)); - getDateRestriction(conversionContext, tablesValidityDate).ifPresent(allSqlFiltersForTable::add); + Optional dateRestriction = getDateRestriction(conversionContext, tablesValidityDate); + dateRestriction.ifPresent(allSqlFiltersForTable::add); // convert selects SelectContext selectContext = SelectContext.create(ids, tablesValidityDate, connectorTables, conversionContext); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CTConditionContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CTConditionContext.java index 4c53669a2a..e6f98f3019 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CTConditionContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CTConditionContext.java @@ -1,22 +1,28 @@ package com.bakdata.conquery.sql.conversion.cqelement.concept; -import com.bakdata.conquery.models.datasets.Column; -import com.bakdata.conquery.models.datasets.Table; +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.name; +import static org.jooq.impl.SQLDataType.VARCHAR; + import com.bakdata.conquery.models.datasets.concepts.Connector; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import lombok.Value; +import org.jooq.Field; @Value public class CTConditionContext { - Table connectorTable; - Column connectorColumn; + public static final Field COLUMN_VALUE_FIELD = field(name("col_val"), VARCHAR(32)); + Field connectorColumn; SqlFunctionProvider functionProvider; - public static CTConditionContext create(Connector connector, SqlFunctionProvider functionProvider) { + public static CTConditionContext forJoinTables(SqlFunctionProvider functionProvider) { + return new CTConditionContext(COLUMN_VALUE_FIELD, functionProvider); + } + + public static CTConditionContext forConnector(Connector connector, SqlFunctionProvider functionProvider) { return new CTConditionContext( - connector.getResolvedTable(), - connector.getColumn() != null ? connector.getColumn().resolve() : null, + connector.getColumn() != null ? field(name(connector.resolveTableId().getTable(), connector.getColumn().getColumn()), VARCHAR(32)) : null, functionProvider ); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java index 99831de70c..a990803c96 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.sql.conversion.dialect; +import static org.jooq.impl.DSL.field; import static org.jooq.impl.DSL.nullif; import java.sql.Date; @@ -42,12 +43,28 @@ public String getAnyCharRegex() { return ANY_CHAR_REGEX; } + + @Override + public Field lower(Field daterange) { + throw new IllegalStateException("HANA does not support DATE_RANGE"); + } + + @Override + public Field upper(Field daterange) { + throw new IllegalStateException("HANA does not support DATE_RANGE"); + } + @Override public Table getNoOpTable() { // see https://help.sap.com/docs/SAP_DATA_HUB/e8d3e271a4554a35a5a6136d3d6af3f8/4d4b939b37b84bea8b2aa2ada640c392.html return DSL.table(DSL.name(NOP_TABLE)); } + @Override + public Field functionParam(String name) { + return field(":" + name); + } + @Override public Condition dateRestriction(ColumnDateRange dateRestriction, ColumnDateRange daterange) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlDialect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlDialect.java index 14571c0cac..cf2f74c218 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlDialect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlDialect.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.sql.conversion.dialect; +import static org.jooq.impl.SQLDataType.NVARCHAR; + import java.util.List; import com.bakdata.conquery.models.events.MajorTypeId; @@ -9,9 +11,12 @@ import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.PostgreSqlIntervalPacker; import com.bakdata.conquery.sql.execution.DefaultSqlCDateSetParser; import com.bakdata.conquery.sql.execution.SqlCDateSetParser; +import lombok.extern.slf4j.Slf4j; import org.jooq.DSLContext; import org.jooq.Field; +import org.postgresql.util.PGmoney; +@Slf4j public class PostgreSqlDialect implements SqlDialect { private final SqlFunctionProvider postgresqlFunctionProvider; @@ -43,15 +48,16 @@ public List> getNodeConverters(DSLContext dsl @Override public boolean isTypeCompatible(Field field, MajorTypeId type) { + log.debug("Field {} type: getTypeName={}, getQualifiedName={}", field.getName(), field.getDataType().getTypeName(), field.getDataType().getQualifiedName()); return switch (type) { case STRING -> field.getDataType().isString(); case INTEGER -> field.getDataType().isInteger(); case BOOLEAN -> field.getDataType().isBoolean(); case REAL -> field.getDataType().isNumeric(); case DECIMAL -> field.getDataType().isDecimal(); - case MONEY -> field.getDataType().isDecimal(); + case MONEY -> true; // TODO Need to find proper name case DATE -> field.getDataType().isDate(); - case DATE_RANGE -> field.getDataType().getTypeName().equals("daterange"); + case DATE_RANGE -> true; // TODO Not properly fetched from postgres field.getDataType().getTypeName().equals("daterange"); }; } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java index 1d49274a89..9c75a1c9c1 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.sql.conversion.dialect; -import static org.jooq.impl.DSL.field; -import static org.jooq.impl.DSL.nullif; +import static org.jooq.impl.DSL.*; import java.sql.Date; import java.time.temporal.ChronoUnit; @@ -58,7 +57,7 @@ public String getAnyCharRegex() { @Override public Table getNoOpTable() { - return DSL.table(DSL.select(DSL.val(1))).as(DSL.name(SharedAliases.NOP_TABLE.getAlias())); + return table(select(val(1))).as(name(SharedAliases.NOP_TABLE.getAlias())); } @NotNull @@ -74,6 +73,10 @@ public Collection> orderByValidityDates( .toList(); } + public Field emptyDateRange() { + return field("{0}::daterange", val("empty")); + } + @Override public String getMinDateExpression() { return MINUS_INFINITY_DATE_VALUE; @@ -82,7 +85,7 @@ public String getMinDateExpression() { @Override public Condition dateRestriction(ColumnDateRange dateRestriction, ColumnDateRange daterange) { // the && operator checks if two ranges overlap (see https://www.postgresql.org/docs/15/functions-range.html) - return DSL.condition( + return condition( "{0} && {1}", ensureIsSingleColumnRange(dateRestriction).getRange(), ensureIsSingleColumnRange(daterange).getRange() @@ -96,12 +99,12 @@ private ColumnDateRange ensureIsSingleColumnRange(ColumnDateRange daterange) { } public Field daterange(Field startColumn, Field endColumn, String bounds) { - return DSL.function( + return function( "daterange", Object.class, startColumn, endColumn, - DSL.val(bounds) + val(bounds) ); } @@ -129,13 +132,13 @@ public ColumnDateRange forCDateRange(CDateRange daterange) { endDateExpression = daterange.getMax().toString(); } - Field daterangeField = daterange(DSL.val(startDateExpression), DSL.val(endDateExpression), CLOSED_RANGE); + Field daterangeField = daterange(val(startDateExpression), val(endDateExpression), CLOSED_RANGE); return ColumnDateRange.of(daterangeField); } private Field datemultirange(Field... fields) { - return DSL.function("datemultirange", Object.class, fields); + return function("datemultirange", Object.class, fields); } @Override @@ -160,7 +163,7 @@ private ColumnDateRange toColumnDateRange(ValidityDate validityDate) { @Override public Field toDateField(String dateValue) { - return DSL.field("{0}::{1}", Date.class, DSL.val(dateValue), DSL.keyword("date")); + return field("{0}::{1}", Date.class, val(dateValue), keyword("date")); } private ColumnDateRange ofSingleColumn(String tableName, Column column) { @@ -170,19 +173,19 @@ private ColumnDateRange ofSingleColumn(String tableName, Column column) { dateRange = switch (column.getType()) { // if validityDateColumn is a DATE_RANGE we can make use of Postgres' integrated daterange type, but the upper bound is exclusive by default case DATE_RANGE -> { - Field daterange = DSL.field(DSL.name(column.getName())); - Field withOpenLowerEnd = DSL.coalesce(lower(daterange), toDateField(MINUS_INFINITY_DATE_VALUE)); - Field withOpenUpperEnd = DSL.coalesce(upper(daterange), toDateField(INFINITY_DATE_VALUE)); - yield DSL.when(daterange.isNull(), emptyDateRange()) - .otherwise(daterange(withOpenLowerEnd, withOpenUpperEnd, OPEN_RANGE)); + Field daterange = field(name(column.getName())); + Field withOpenLowerEnd = coalesce(lower(daterange), toDateField(MINUS_INFINITY_DATE_VALUE)); + Field withOpenUpperEnd = coalesce(upper(daterange), toDateField(INFINITY_DATE_VALUE)); + yield when(daterange.isNull(), emptyDateRange()) + .otherwise(daterange(withOpenLowerEnd, withOpenUpperEnd, OPEN_RANGE)); } // if the validity date column is not of daterange type, we construct it manually case DATE -> { - Field singleDate = DSL.field(DSL.name(tableName, column.getName()), Date.class); - Field withOpenLowerEnd = DSL.coalesce(singleDate, toDateField(MINUS_INFINITY_DATE_VALUE)); - Field withOpenUpperEnd = DSL.coalesce(singleDate, toDateField(INFINITY_DATE_VALUE)); - yield DSL.when(singleDate.isNull(), emptyDateRange()) - .otherwise(daterange(withOpenLowerEnd, withOpenUpperEnd, CLOSED_RANGE)); + Field singleDate = field(name(tableName, column.getName()), Date.class); + Field withOpenLowerEnd = coalesce(singleDate, toDateField(MINUS_INFINITY_DATE_VALUE)); + Field withOpenUpperEnd = coalesce(singleDate, toDateField(INFINITY_DATE_VALUE)); + yield when(singleDate.isNull(), emptyDateRange()) + .otherwise(daterange(withOpenLowerEnd, withOpenUpperEnd, CLOSED_RANGE)); } default -> throw new IllegalArgumentException( "Given column type '%s' can't be converted to a proper date restriction.".formatted(column.getType()) @@ -194,27 +197,28 @@ private ColumnDateRange ofSingleColumn(String tableName, Column column) { private ColumnDateRange ofStartAndEnd(String tableName, Column startColumn, Column endColumn) { - Field startField = DSL.field(DSL.name(tableName, startColumn.getName())); - Field withOpenLowerEnd = DSL.coalesce(startField, toDateField(MINUS_INFINITY_DATE_VALUE)); - Field endField = DSL.field(DSL.name(tableName, endColumn.getName())); - Field withOpenUpperEnd = DSL.coalesce(endField, toDateField(INFINITY_DATE_VALUE)); + Field startField = field(name(tableName, startColumn.getName())); + Field withOpenLowerEnd = coalesce(startField, toDateField(MINUS_INFINITY_DATE_VALUE)); + Field endField = field(name(tableName, endColumn.getName())); + Field withOpenUpperEnd = coalesce(endField, toDateField(INFINITY_DATE_VALUE)); return ColumnDateRange.of( - DSL.when(startField.isNull().and(endField.isNull()), emptyDateRange()) - .otherwise(this.daterange(withOpenLowerEnd, withOpenUpperEnd, CLOSED_RANGE)) + when(startField.isNull().and(endField.isNull()), emptyDateRange()) + .otherwise(this.daterange(withOpenLowerEnd, withOpenUpperEnd, CLOSED_RANGE)) ); } - private static Field lower(Field daterange) { - return DSL.function("lower", Date.class, daterange); + public Field lower(Field daterange) { + return function("lower", Date.class, daterange); } - private static Field upper(Field daterange) { - return DSL.function("upper", Date.class, daterange); + public Field upper(Field daterange) { + return function("upper", Date.class, daterange); } - public Field emptyDateRange() { - return DSL.field("{0}::daterange", DSL.val("empty")); + @Override + public Field functionParam(String name) { + return field(name(name)); } @Override @@ -242,7 +246,7 @@ private ColumnDateRange toColumnDateRange(CDateRange dateRestriction) { @Override public ColumnDateRange intersection(ColumnDateRange left, ColumnDateRange right) { - return ColumnDateRange.of(DSL.field( + return ColumnDateRange.of(field( "{0} * {1}", ensureIsSingleColumnRange(left).getRange(), ensureIsSingleColumnRange(right).getRange() @@ -266,14 +270,14 @@ public ColumnDateRange aggregated(ColumnDateRange columnDateRange) { } private Field rangeAgg(ColumnDateRange columnDateRange) { - return DSL.function("range_agg", Object.class, columnDateRange.getRange()); + return function("range_agg", Object.class, columnDateRange.getRange()); } @Override public ColumnDateRange toDualColumn(ColumnDateRange columnDateRange) { Field daterange = columnDateRange.getRange(); - Field start = DSL.function("lower", Date.class, daterange); - Field end = DSL.function("upper", Date.class, daterange); + Field start = function("lower", Date.class, daterange); + Field end = function("upper", Date.class, daterange); return ColumnDateRange.of(start, end); } @@ -296,7 +300,7 @@ public QueryStep unnestDaterange(ColumnDateRange nested, QueryStep predecessor, } private static Field unnest(Field multirange) { - return DSL.function("unnest", Object.class, multirange); + return function("unnest", Object.class, multirange); } @Override @@ -310,7 +314,7 @@ public Field daterangeStringExpression(ColumnDateRange columnDateRange) if (!columnDateRange.isSingleColumnRange()) { throw new UnsupportedOperationException("All column date ranges should have been converted to single column ranges."); } - Field aggregatedValidityDate = DSL.field("({0})::{1}", String.class, columnDateRange.getRange(), DSL.keyword("varchar")); + Field aggregatedValidityDate = field("({0})::{1}", String.class, columnDateRange.getRange(), keyword("varchar")); return replace(aggregatedValidityDate, INFINITY_DATE_VALUE, INFINITY_SIGN); } @@ -321,7 +325,7 @@ public Field dateDistance(ChronoUnit datePart, Field startDate, F return cast(endDate.minus(startDate), SQLDataType.INTEGER); } - Field age = DSL.function("age", Integer.class, endDate, startDate); + Field age = function("age", Integer.class, endDate, startDate); return switch (datePart) { case MONTHS -> extract(DatePart.YEAR, age).multiply(12).plus(extract(DatePart.MONTH, age)); case YEARS -> extract(DatePart.YEAR, age); @@ -337,12 +341,12 @@ public Field cast(Field field, DataType type) { } public Field extract(DatePart datePart, Field timeInterval) { - return DSL.field( + return field( "{0}({1} {2} {3})", Integer.class, - DSL.keyword("extract"), - DSL.keyword(datePart.toSQL()), - DSL.keyword("from"), + keyword("extract"), + keyword(datePart.toSQL()), + keyword("from"), timeInterval ); } @@ -354,13 +358,13 @@ public Field addDays(Field dateColumn, Field amountOfDays) @Override public Field random(Field column) { - ArrayAggOrderByStep arrayAgg = DSL.arrayAgg(DSL.field( + ArrayAggOrderByStep arrayAgg = arrayAgg(field( "{0} {1} {2}", column, - DSL.keyword("ORDER BY"), - DSL.function("random", Object.class) + keyword("ORDER BY"), + function("random", Object.class) )); - return DSL.field("({0})[1]", column.getType(), arrayAgg); + return field("({0})[1]", column.getType(), arrayAgg); } @Override @@ -370,7 +374,7 @@ public Condition likeRegex(Field field, String pattern) { @Override public Field yearQuarter(Field dateField) { - return DSL.field( + return field( "{0}::varchar || '-Q' || {1}::varchar", String.class, DSL.extract(dateField, DatePart.YEAR), diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java index c8b606088a..422b594dfe 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java @@ -1,11 +1,12 @@ package com.bakdata.conquery.sql.conversion.dialect; +import static org.jooq.impl.DSL.function; + import java.sql.Date; import java.time.temporal.ChronoUnit; import java.util.Collection; import java.util.List; -import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; @@ -111,7 +112,7 @@ Collection> orderByValidityDates( * @param predecessor The predeceasing step containing the aggregated {@link ColumnDateRange}. * @param nested The {@link ColumnDateRange} you want to unnest. * @param cteName The CTE name of the returned {@link QueryStep}. - * @return A QueryStep containing an unnested validity date with 1 row per single daterange for each id. For dialects that don't support single column + * @return A QueryStep containing an unnested validity date with 1 row per single daterange for each conceptElement. For dialects that don't support single column * multiranges, the given predecessor will be returned as is. */ QueryStep unnestDaterange(ColumnDateRange nested, QueryStep predecessor, String cteName); @@ -229,8 +230,7 @@ default Field prefixStringAggregation(Field field, String prefix ); } - default - Condition validityDateFilter(ValidityDate validityDate) { + default Condition validityDateFilter(ValidityDate validityDate) { if (validityDate.isSingleColumnDaterange()) { Column column = validityDate.getColumn().resolve(); @@ -245,4 +245,14 @@ Condition validityDateFilter(ValidityDate validityDate) { ); } + default Field lower(Field daterange) { + return function("lower", Date.class, daterange); + } + + default Field upper(Field daterange) { + return function("upper", Date.class, daterange); + } + + Field functionParam(String name); + } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java index d151fc6b93..ecf31c729f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java @@ -46,7 +46,7 @@ public QueryStep createStratificationTable(List rowNumber = DSL.rowNumber().over().coerce(Object.class); + Field rowNumber = DSL.rowNumber().over().coerce(String.class); SqlIdColumns ids = new SqlIdColumns(rowNumber); FieldWrapper seriesIndex = new FieldWrapper<>(stratificationFunctions.intSeriesField()); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ColumnDateRange.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ColumnDateRange.java index 5cc417d988..62dba4f799 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ColumnDateRange.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ColumnDateRange.java @@ -24,16 +24,16 @@ public class ColumnDateRange implements SqlSelect { private final String alias; protected ColumnDateRange(Field startColumn, Field endColumn, String alias) { - this.range = null; - this.start = startColumn; - this.end = endColumn; + range = null; + start = startColumn; + end = endColumn; this.alias = alias; } protected ColumnDateRange(Field range, String alias) { this.range = range; - this.start = null; - this.end = null; + start = null; + end = null; this.alias = alias; } @@ -54,12 +54,12 @@ public static ColumnDateRange of(Field startColumn, Field endColumn, } public static ColumnDateRange empty() { - Field emptyRange = DSL.field(DSL.val("{}")); + final Field emptyRange = DSL.field(DSL.val("{}")); return ColumnDateRange.of(emptyRange); } public ColumnDateRange asValidityDateRange(String alias) { - return this.as(alias + VALIDITY_DATE_COLUMN_NAME_SUFFIX); + return as(alias + VALIDITY_DATE_COLUMN_NAME_SUFFIX); } /** @@ -67,15 +67,15 @@ public ColumnDateRange asValidityDateRange(String alias) { * False if it consists of a start and end field. */ public boolean isSingleColumnRange() { - return this.range != null; + return range != null; } @Override public List> toFields() { if (isSingleColumnRange()) { - return List.of(this.range); + return List.of(range); } - return Stream.of(this.start, this.end) + return Stream.of(start, end) .collect(Collectors.toList()); } @@ -98,43 +98,43 @@ public List requiredColumns() { public ColumnDateRange as(String alias) { if (isSingleColumnRange()) { - return new ColumnDateRange(this.range.as(alias), alias); + return new ColumnDateRange(range.as(alias), alias); } return new ColumnDateRange( - this.start.as(alias + START_SUFFIX), - this.end.as(alias + END_SUFFIX), + start.as(alias + START_SUFFIX), + end.as(alias + END_SUFFIX), alias ); } public ColumnDateRange coalesce(ColumnDateRange right) { - if (this.isSingleColumnRange() != right.isSingleColumnRange()) { + if (isSingleColumnRange() != right.isSingleColumnRange()) { throw new UnsupportedOperationException("Can only join ColumnDateRanges of same type"); } if (isSingleColumnRange()) { - return ColumnDateRange.of(DSL.coalesce(this.range, right.getRange())).as(this.alias); + return ColumnDateRange.of(DSL.coalesce(range, right.getRange())).as(alias); } return ColumnDateRange.of( - DSL.coalesce(this.start, right.getStart()), - DSL.coalesce(this.end, right.getEnd()) - ).as(this.alias); + DSL.coalesce(start, right.getStart()), + DSL.coalesce(end, right.getEnd()) + ).as(alias); } public Condition join(ColumnDateRange right) { - if (this.isSingleColumnRange() != right.isSingleColumnRange()) { + if (isSingleColumnRange() != right.isSingleColumnRange()) { throw new UnsupportedOperationException("Can only join ColumnDateRanges of same type"); } - if (this.isSingleColumnRange()) { - return this.range.coerce(Object.class).eq(right.getRange()); + if (isSingleColumnRange()) { + return range.coerce(Object.class).eq(right.getRange()); } - return this.start.eq(right.getStart()).and(end.eq(right.getEnd())); + return start.eq(right.getStart()).and(end.eq(right.getEnd())); } public Condition isNotNull() { - if (this.isSingleColumnRange()) { - return this.range.isNotNull(); + if (isSingleColumnRange()) { + return range.isNotNull(); } - return this.start.isNotNull().and(this.end.isNotNull()); + return start.isNotNull().and(end.isNotNull()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlIdColumns.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlIdColumns.java index f80ebf8d35..899b64050c 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlIdColumns.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlIdColumns.java @@ -23,21 +23,21 @@ public class SqlIdColumns implements Qualifiable { @Getter - private final Field primaryColumn; + private final Field primaryColumn; @Nullable - private final Field secondaryId; + private final Field secondaryId; @Nullable private final SqlIdColumns predecessor; - public SqlIdColumns(Field primaryColumn, Field secondaryId) { + public SqlIdColumns(Field primaryColumn, Field secondaryId) { this.primaryColumn = primaryColumn; this.secondaryId = secondaryId; this.predecessor = null; } - public SqlIdColumns(Field primaryColumn) { + public SqlIdColumns(Field primaryColumn) { this.primaryColumn = primaryColumn; this.secondaryId = null; this.predecessor = null; @@ -56,11 +56,11 @@ public SqlIdColumns withAlias() { @Override public SqlIdColumns qualify(String qualifier) { - Field primaryColumn = QualifyingUtil.qualify(this.primaryColumn, qualifier); + Field primaryColumn = QualifyingUtil.qualify(this.primaryColumn, qualifier); if (secondaryId == null) { return new SqlIdColumns(primaryColumn, null, this); } - Field secondaryId = QualifyingUtil.qualify(this.secondaryId, qualifier); + Field secondaryId = QualifyingUtil.qualify(this.secondaryId, qualifier); return new SqlIdColumns(primaryColumn, secondaryId, this); } @@ -90,7 +90,7 @@ public SqlIdColumns forFinalSelect() { return this; } - public Optional> getSecondaryId() { + public Optional> getSecondaryId() { return Optional.ofNullable(this.secondaryId); } @@ -103,7 +103,11 @@ public boolean isWithStratification() { } public List> toFields() { - return Stream.concat(Stream.of(this.primaryColumn), Optional.ofNullable(this.secondaryId).stream()).collect(Collectors.toList()); + if (getSecondaryId().isEmpty()){ + return List.of(getPrimaryColumn()); + } + + return List.of(getPrimaryColumn(), getSecondaryId().get()); } public List join(SqlIdColumns rightIds) { @@ -121,8 +125,8 @@ public List join(SqlIdColumns rightIds) { public SqlIdColumns coalesce(List selectsIds) { - List> primaryColumns = new ArrayList<>(); - List> secondaryIds = new ArrayList<>(); + List> primaryColumns = new ArrayList<>(); + List> secondaryIds = new ArrayList<>(); // add this ids primaryColumns.add(this.primaryColumn); @@ -134,20 +138,23 @@ public SqlIdColumns coalesce(List selectsIds) { ids.getSecondaryId().ifPresent(secondaryIds::add); }); - Field coalescedPrimaryColumn = coalesceFields(primaryColumns).as(SharedAliases.PRIMARY_COLUMN.getAlias()); + Field coalescedPrimaryColumn = coalesceFields(primaryColumns, String.class).as(SharedAliases.PRIMARY_COLUMN.getAlias()); if (secondaryIds.isEmpty()) { return new SqlIdColumns(coalescedPrimaryColumn); } - Field coalescedSecondaryIds = coalesceFields(secondaryIds).as(SharedAliases.SECONDARY_ID.getAlias()); + Field coalescedSecondaryIds = coalesceFields(secondaryIds, String.class).as(SharedAliases.SECONDARY_ID.getAlias()); return new SqlIdColumns(coalescedPrimaryColumn, coalescedSecondaryIds); } - protected static Field coalesceFields(List> fields) { - if (fields.size() == 1) { - return fields.get(0).coerce(Object.class); + protected static Field coalesceFields(List> fields, Class type) { + Field out = fields.getFirst().coerce(type); + + for (int index = 1; index < fields.size(); index++) { + out = DSL.coalesce(out, fields.get(index).coerce(type)); } - return DSL.coalesce(fields.get(0), fields.subList(1, fields.size()).toArray()); + + return out; } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/StratificationSqlIdColumns.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/StratificationSqlIdColumns.java index ba392784c8..c24be4464e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/StratificationSqlIdColumns.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/StratificationSqlIdColumns.java @@ -35,7 +35,7 @@ class StratificationSqlIdColumns extends SqlIdColumns { @Override public SqlIdColumns qualify(String qualifier) { - Field primaryColumn = QualifyingUtil.qualify(getPrimaryColumn(), qualifier); + Field primaryColumn = QualifyingUtil.qualify(getPrimaryColumn(), qualifier); Field resolution = QualifyingUtil.qualify(this.resolution, qualifier); Field index = QualifyingUtil.qualify(this.index, qualifier); Field eventDate = null; @@ -124,9 +124,9 @@ public SqlIdColumns coalesce(List selectsIds) { "Can only coalesce SqlIdColumns if all are with stratification" ); - List> primaryColumns = new ArrayList<>(); - List> resolutions = new ArrayList<>(); - List> indices = new ArrayList<>(); + List> primaryColumns = new ArrayList<>(); + List> resolutions = new ArrayList<>(); + List> indices = new ArrayList<>(); List> eventDates = new ArrayList<>(); // add this ids @@ -147,12 +147,12 @@ public SqlIdColumns coalesce(List selectsIds) { } } - Field coalescedPrimaryColumn = coalesceFields(primaryColumns).as(SharedAliases.PRIMARY_COLUMN.getAlias()); - Field coalescedResolutions = coalesceFields(resolutions).coerce(String.class).as(SharedAliases.RESOLUTION.getAlias()); - Field coalescedIndices = coalesceFields(indices).coerce(Integer.class).as(SharedAliases.INDEX.getAlias()); + Field coalescedPrimaryColumn = coalesceFields(primaryColumns, String.class).as(SharedAliases.PRIMARY_COLUMN.getAlias()); + Field coalescedResolutions = coalesceFields(resolutions, String.class).as(SharedAliases.RESOLUTION.getAlias()); + Field coalescedIndices = coalesceFields(indices, Integer.class).as(SharedAliases.INDEX.getAlias()); Field eventDate = null; if (!eventDates.isEmpty()) { - eventDate = coalesceFields(eventDates).coerce(Date.class).as(SharedAliases.INDEX_SELECTOR.getAlias()); + eventDate = coalesceFields(eventDates, Date.class).as(SharedAliases.INDEX_SELECTOR.getAlias()); } return StratificationSqlIdColumns.builder() diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java index 6b1878f0c5..dbe5df16c4 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java @@ -115,7 +115,7 @@ private static QueryStep convertTable( Map positions, ConversionContext context ) { - final Field primaryColumn = TablePrimaryColumnUtil.findPrimaryColumn(cqTable.getConnector().resolve().getResolvedTable(), context.getConfig()); + final Field primaryColumn = TablePrimaryColumnUtil.findPrimaryColumn(cqTable.getConnector().resolve().getResolvedTable(), context.getConfig()); final SqlIdColumns ids = new SqlIdColumns(primaryColumn); final String conceptConnectorName = context.getNameGenerator().conceptConnectorName(concept, cqTable.getConnector().resolve(), context.getSqlPrintSettings().getLocale()); diff --git a/backend/src/main/java/com/bakdata/conquery/util/TablePrimaryColumnUtil.java b/backend/src/main/java/com/bakdata/conquery/util/TablePrimaryColumnUtil.java index 685bdf254f..d9187ef3be 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/TablePrimaryColumnUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/util/TablePrimaryColumnUtil.java @@ -1,17 +1,24 @@ package com.bakdata.conquery.util; +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.name; + import com.bakdata.conquery.models.config.DatabaseConfig; import com.bakdata.conquery.models.datasets.Table; import org.jooq.Field; -import org.jooq.impl.DSL; public class TablePrimaryColumnUtil { - public static Field findPrimaryColumn(Table table, DatabaseConfig databaseConfig) { - String primaryColumnName = table.getPrimaryColumn() == null - ? databaseConfig.getPrimaryColumn() - : table.getPrimaryColumn().getName(); - return DSL.field(DSL.name(table.getName(), primaryColumnName)); + public static Field findPrimaryColumn(Table table, DatabaseConfig databaseConfig) { + String primaryColumnName; + if (table.getPrimaryColumn() != null) { + primaryColumnName = table.getPrimaryColumn().getName(); + } + else { + primaryColumnName = databaseConfig.getPrimaryColumn(); + } + + return field(name(table.getName(), primaryColumnName), String.class); } } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java b/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java index e42696b539..6ba0ce7f3a 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java @@ -230,8 +230,6 @@ public static void importCqppFiles(StandaloneSupport support, List cqppFil } support.waitUntilWorkDone(); - - } public static void uploadCqpp(StandaloneSupport support, File cqpp, boolean update, Response.Status.Family expectedResponseFamily) { diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/SqlTestDataImporter.java b/backend/src/test/java/com/bakdata/conquery/integration/json/SqlTestDataImporter.java index 5d3ec9acf4..ae9f0c04d7 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/SqlTestDataImporter.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/SqlTestDataImporter.java @@ -3,6 +3,7 @@ import java.util.Collection; import java.util.List; +import com.bakdata.conquery.integration.common.LoadingUtil; import com.bakdata.conquery.integration.common.RequiredData; import com.bakdata.conquery.integration.common.RequiredTable; import com.bakdata.conquery.integration.json.filter.FilterTest; @@ -32,6 +33,8 @@ public void importQueryTestData(StandaloneSupport support, QueryTest test) throw importSearchIndexes(support, test.getSearchIndexes()); importIdMapping(support, content); + waitUntilDone(support, () -> LoadingUtil.updateMatchingStats(support)); + } @Override @@ -44,6 +47,8 @@ public void importFormTestData(StandaloneSupport support, FormTest test) throws importTableContents(support, content.getTables()); importIdMapping(support, content); importPreviousQueries(support, content); + waitUntilDone(support, () -> LoadingUtil.updateMatchingStats(support)); + } @Override diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/CsvTableImporter.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/CsvTableImporter.java index 8c3ab27b3d..eabd12fc4d 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/sql/CsvTableImporter.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/CsvTableImporter.java @@ -149,7 +149,7 @@ private void insertValuesIntoTable(Table table, List> columns, if (content.isEmpty()) { return; } - log.debug("Inserting into table: {}", content); + log.trace("Inserting into table: {}", content); testSqlDialect.getTestFunctionProvider().insertValuesIntoTable(table, columns, content, statement, dslContext); } diff --git a/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/tree/MatchingStatsTests.java b/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/tree/MatchingStatsTests.java index 8938a83e1c..73918bb68c 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/tree/MatchingStatsTests.java +++ b/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/tree/MatchingStatsTests.java @@ -2,8 +2,6 @@ import static org.assertj.core.api.Assertions.assertThat; -import com.bakdata.conquery.models.datasets.Column; -import com.bakdata.conquery.models.datasets.Table; import com.bakdata.conquery.models.datasets.concepts.MatchingStats; import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; import com.bakdata.conquery.models.identifiable.ids.specific.WorkerId; @@ -11,80 +9,74 @@ public class MatchingStatsTests { - private final WorkerId workerId1 = new WorkerId(new DatasetId("sampleDataset"), "sampleWorker"); - private final WorkerId workerId2 = new WorkerId(new DatasetId("sampleDataset2"), "sampleWorker2"); + private final WorkerId workerId1 = new WorkerId(new DatasetId("sampleDataset"), "sampleWorker"); + private final WorkerId workerId2 = new WorkerId(new DatasetId("sampleDataset2"), "sampleWorker2"); - @Test - public void entitiesCountTest() { + @Test + public void entitiesCountTest() { - MatchingStats stats = new MatchingStats(); + MatchingStats stats = new MatchingStats(); - assertThat(stats.countEntities()).isEqualTo(0); + assertThat(stats.countEntities()).isEqualTo(0); - stats.putEntry(workerId1, new MatchingStats.Entry(5, 5, 10, 20)); - assertThat(stats.countEntities()).isEqualTo(5); + stats.putEntry(workerId1.toString(), new MatchingStats.Entry(5, 5, 10, 20)); + assertThat(stats.countEntities()).isEqualTo(5); - stats.putEntry(workerId1, new MatchingStats.Entry(5, 8, 10, 20)); - assertThat(stats.countEntities()).isEqualTo(8); + stats.putEntry(workerId1.toString(), new MatchingStats.Entry(5, 8, 10, 20)); + assertThat(stats.countEntities()).isEqualTo(8); - stats.putEntry(workerId2, new MatchingStats.Entry(5, 2, 10, 20)); - assertThat(stats.countEntities()).isEqualTo(10); + stats.putEntry(workerId2.toString(), new MatchingStats.Entry(5, 2, 10, 20)); + assertThat(stats.countEntities()).isEqualTo(10); - } + } - @Test - public void addEventTest(){ - MatchingStats stats = new MatchingStats(); - Table table = new Table(); - table.setColumns(new Column[0]); + @Test + public void addEventTest() { + MatchingStats stats = new MatchingStats(); - assertThat(stats.countEvents()).isEqualTo(0); - assertThat(stats.countEntities()).isEqualTo(0); + assertThat(stats.countEvents()).isEqualTo(0); + assertThat(stats.countEntities()).isEqualTo(0); - MatchingStats.Entry entry1 = new MatchingStats.Entry(); - entry1.addEvent(table, null, 1, "1"); - entry1.addEvent(table, null, 2, "1"); - entry1.addEvent(table, null, 3, "2"); - entry1.addEvent(table, null, 4, "2"); + MatchingStats.Entry entry1 = new MatchingStats.Entry(); + entry1.addEvents("1", 1, null); + entry1.addEvents("1", 1, null); - entry1.addEvent(table, null, 5, "3"); - entry1.addEvent(table, null, 6, "3"); + entry1.addEvents("2", 1, null); + entry1.addEvents("2", 1, null); - entry1.addEvent(table, null, 7, "4"); - entry1.addEvent(table, null, 8, "4"); + entry1.addEvents("3", 1, null); + entry1.addEvents("3", 1, null); + entry1.addEvents("4", 1, null); + entry1.addEvents("4", 1, null); - stats.putEntry(workerId1, entry1); - assertThat(stats.countEvents()).isEqualTo(8); - assertThat(stats.countEntities()).isEqualTo(4); + stats.putEntry(workerId1.toString(), entry1); + assertThat(stats.countEvents()).isEqualTo(8); + assertThat(stats.countEntities()).isEqualTo(4); - MatchingStats.Entry entry2 = new MatchingStats.Entry(); + MatchingStats.Entry entry2 = new MatchingStats.Entry(); - entry2.addEvent(table, null, 1, "1"); - entry2.addEvent(table, null, 2, "2"); + entry2.addEvents("1", 1, null); + entry2.addEvents("2", 1, null); + entry2.addEvents("3", 1, null); + entry2.addEvents("4", 1, null); + entry2.addEvents("5", 1, null); + entry2.addEvents("6", 1, null); + entry2.addEvents("7", 1, null); + entry2.addEvents("8", 1, null); + entry2.addEvents("9", 1, null); + entry2.addEvents("10", 1, null); - entry2.addEvent(table, null, 3, "3"); - entry2.addEvent(table, null, 4, "4"); - entry2.addEvent(table, null, 5, "5"); - entry2.addEvent(table, null, 6, "6"); + stats.putEntry(workerId2.toString(), entry2); + assertThat(stats.countEvents()).isEqualTo(18); + assertThat(stats.countEntities()).isEqualTo(14); - entry2.addEvent(table, null, 7, "7"); - entry2.addEvent(table, null, 8, "8"); - entry2.addEvent(table, null, 9, "9"); - entry2.addEvent(table, null, 10, "10"); - - stats.putEntry(workerId2, entry2); - assertThat(stats.countEvents()).isEqualTo(18); - assertThat(stats.countEntities()).isEqualTo(14); - - - - } + } } diff --git a/backend/src/test/resources/tests/form/shared/abc.concept.json b/backend/src/test/resources/tests/form/shared/abc.concept.json index c0791d21e0..e7552d9d23 100644 --- a/backend/src/test/resources/tests/form/shared/abc.concept.json +++ b/backend/src/test/resources/tests/form/shared/abc.concept.json @@ -28,16 +28,16 @@ { "name": "a", "condition": { - "type": "PREFIX_LIST", - "prefixes": "A" + "type": "EQUAL", + "values": ["A"] }, "children": [] }, { "name": "b", "condition": { - "type": "PREFIX_LIST", - "prefixes": "B" + "type": "EQUAL", + "values": ["B"] }, "children": [] } diff --git a/backend/src/test/resources/tests/sql/combined/combined.json b/backend/src/test/resources/tests/sql/combined/combined.json index 119b3c3f58..6fc0f1b719 100644 --- a/backend/src/test/resources/tests/sql/combined/combined.json +++ b/backend/src/test/resources/tests/sql/combined/combined.json @@ -136,28 +136,6 @@ } ], "children": [ - { - "label": "test_child1", - "description": " ", - "condition": { - "type": "EQUAL", - "values": [ - "A1" - ] - }, - "children": [] - }, - { - "label": "test_child2", - "description": " ", - "condition": { - "type": "EQUAL", - "values": [ - "B2" - ] - }, - "children": [] - } ], "selects": [ { diff --git a/backend/src/test/resources/tests/sql/multiple_tables/multiple_tables.spec.json b/backend/src/test/resources/tests/sql/multiple_tables/multiple_tables.spec.json index f1d660abfc..360c85da24 100644 --- a/backend/src/test/resources/tests/sql/multiple_tables/multiple_tables.spec.json +++ b/backend/src/test/resources/tests/sql/multiple_tables/multiple_tables.spec.json @@ -1,7 +1,7 @@ { "type": "QUERY_TEST", "sqlSpec": { - "isEnabled": true + "isEnabled": false }, "label": "MULTIPLE_TABLES_ICD_QUERY test", "expectedCsv": "tests/sql/multiple_tables/expected.csv", diff --git a/backend/src/test/resources/tests/sql/selects/concept_values/single_connector.json b/backend/src/test/resources/tests/sql/selects/concept_values/single_connector.json index dd9193c671..403d6940a5 100644 --- a/backend/src/test/resources/tests/sql/selects/concept_values/single_connector.json +++ b/backend/src/test/resources/tests/sql/selects/concept_values/single_connector.json @@ -53,16 +53,16 @@ { "label": "test_child1", "condition": { - "type": "PREFIX_LIST", - "prefixes": "A" + "type": "EQUAL", + "values": "A1" }, "children": [] }, { "label": "test_child2", "condition": { - "type": "PREFIX_LIST", - "prefixes": "B" + "type": "EQUAL", + "values": ["B2"] }, "children": [] } diff --git a/backend/src/test/resources/tests/sql/selects/concept_values/two_connectors.json b/backend/src/test/resources/tests/sql/selects/concept_values/two_connectors.json index 746ca1563f..1ccaf4a4d4 100644 --- a/backend/src/test/resources/tests/sql/selects/concept_values/two_connectors.json +++ b/backend/src/test/resources/tests/sql/selects/concept_values/two_connectors.json @@ -56,16 +56,16 @@ { "label": "test_child1", "condition": { - "type": "PREFIX_LIST", - "prefixes": "A" + "type": "EQUAL", + "values": ["A1"] }, "children": [] }, { "label": "test_child2", "condition": { - "type": "PREFIX_LIST", - "prefixes": "B" + "type": "EQUAL", + "values": ["B2"] }, "children": [] } diff --git a/backend/src/test/resources/tests/sql/selects/sum/duration_sum/duration_sum.json b/backend/src/test/resources/tests/sql/selects/sum/duration_sum/duration_sum.json index b6862fc9e0..3b0e47d31c 100644 --- a/backend/src/test/resources/tests/sql/selects/sum/duration_sum/duration_sum.json +++ b/backend/src/test/resources/tests/sql/selects/sum/duration_sum/duration_sum.json @@ -55,16 +55,16 @@ { "name": "a", "condition": { - "type": "PREFIX_LIST", - "prefixes": "A" + "type": "EQUAL", + "values": ["A"] }, "children": [] }, { "name": "b", "condition": { - "type": "PREFIX_LIST", - "prefixes": "B" + "type": "EQUAL", + "values": ["B"] }, "children": [] } diff --git a/backend/src/test/resources/tests/sql/selects/sum/event_duration_sum/duration_sum.json b/backend/src/test/resources/tests/sql/selects/sum/event_duration_sum/duration_sum.json index 8e9a14690d..e33d8989b7 100644 --- a/backend/src/test/resources/tests/sql/selects/sum/event_duration_sum/duration_sum.json +++ b/backend/src/test/resources/tests/sql/selects/sum/event_duration_sum/duration_sum.json @@ -58,16 +58,16 @@ { "name": "a", "condition": { - "type": "PREFIX_LIST", - "prefixes": "A" + "type": "EQUAL", + "values": ["A"] }, "children": [] }, { "name": "b", "condition": { - "type": "PREFIX_LIST", - "prefixes": "B" + "type": "EQUAL", + "values": ["B"] }, "children": [] } diff --git a/backend/src/test/resources/tests/sql/tree/nested/nested.spec.json b/backend/src/test/resources/tests/sql/tree/nested/nested.spec.json index 203112d673..20ca58e8b6 100644 --- a/backend/src/test/resources/tests/sql/tree/nested/nested.spec.json +++ b/backend/src/test/resources/tests/sql/tree/nested/nested.spec.json @@ -1,7 +1,7 @@ { "type": "QUERY_TEST", "sqlSpec": { - "isEnabled": true + "isEnabled": false }, "label": "TREE concept with multiple nested conditions", "expectedCsv": "tests/sql/tree/nested/expected.csv", diff --git a/backend/src/test/resources/tests/sql/tree/prefix_range/prefix_range.spec.json b/backend/src/test/resources/tests/sql/tree/prefix_range/prefix_range.spec.json index da4e9e586a..3c797eddef 100644 --- a/backend/src/test/resources/tests/sql/tree/prefix_range/prefix_range.spec.json +++ b/backend/src/test/resources/tests/sql/tree/prefix_range/prefix_range.spec.json @@ -1,7 +1,7 @@ { "type": "QUERY_TEST", "sqlSpec": { - "isEnabled": true + "isEnabled": false }, "label": "PREFIX_RANGE condition test", "expectedCsv": "tests/sql/tree/prefix_range/expected.csv", diff --git a/backend/src/test/resources/tests/sql/tree/with_parent/with_parent.spec.json b/backend/src/test/resources/tests/sql/tree/with_parent/with_parent.spec.json index 7610cfde25..2ef189c244 100644 --- a/backend/src/test/resources/tests/sql/tree/with_parent/with_parent.spec.json +++ b/backend/src/test/resources/tests/sql/tree/with_parent/with_parent.spec.json @@ -1,7 +1,7 @@ { "type": "QUERY_TEST", "sqlSpec": { - "isEnabled": true + "isEnabled": false }, "label": "Tree concept resolving a deep child and it's parents", "expectedCsv": "tests/sql/tree/with_parent/expected.csv",