MID-10990, MID-11046, MID-11047: Fix CSV export performance and stability issues

gui/admin-gui/src/main/java/com/evolveum/midpoint/gui/api/component/button/CsvDownloadButtonPanel.java

@@ -13,12 +13,12 @@
 import com.evolveum.midpoint.gui.api.util.WebModelServiceUtils;
 import com.evolveum.midpoint.prism.Referencable;
 import com.evolveum.midpoint.gui.impl.component.data.provider.SelectableBeanContainerDataProvider;
+import com.evolveum.midpoint.gui.impl.component.data.provider.StreamingCsvDataExporter;
 
 import org.apache.commons.lang3.StringUtils;
 import org.apache.wicket.ajax.AjaxRequestTarget;
 import org.apache.wicket.extensions.markup.html.repeater.data.table.DataTable;
 import org.apache.wicket.extensions.markup.html.repeater.data.table.IColumn;
-import org.apache.wicket.extensions.markup.html.repeater.data.table.export.CSVDataExporter;
 import org.apache.wicket.extensions.markup.html.repeater.data.table.export.ExportToolbar;
 import org.apache.wicket.extensions.markup.html.repeater.data.table.export.IExportableColumn;
 import org.apache.wicket.markup.repeater.data.IDataProvider;
@@ -64,7 +64,7 @@ public CsvDownloadButtonPanel(String id) {
     private static final long serialVersionUID = 1L;
 
     private void initLayout() {
-        CSVDataExporter csvDataExporter = new CSVDataExporter() {
+        StreamingCsvDataExporter csvDataExporter = new StreamingCsvDataExporter(getPageBase()) {
             private static final long serialVersionUID = 1L;
 
             @Override
@@ -116,10 +116,16 @@ public IResourceStream getResourceStream() {
             }
 
             public String getFileName() {
+                String fileName;
                 if (StringUtils.isEmpty(name.getObject())) {
-                    return CsvDownloadButtonPanel.this.getFilename();
+                    fileName = CsvDownloadButtonPanel.this.getFilename();
+                } else {
+                    fileName = name.getObject();
+                }
+                if (!fileName.toLowerCase().endsWith(".csv")) {
+                    fileName += ".csv";
                 }
-                return name.getObject();
+                return fileName;
             }
         };
 

gui/admin-gui/src/main/java/com/evolveum/midpoint/gui/impl/component/data/provider/BaseSearchDataProvider.java

@@ -16,7 +16,9 @@
 import com.evolveum.midpoint.schema.GetOperationOptions;
 import com.evolveum.midpoint.schema.SelectorOptions;
 
+import com.evolveum.midpoint.schema.ObjectHandler;
 import com.evolveum.midpoint.schema.result.OperationResult;
+import com.evolveum.midpoint.task.api.Task;
 import com.evolveum.midpoint.util.exception.CommonException;
 import com.evolveum.midpoint.util.exception.SystemException;
 
@@ -31,7 +33,8 @@
  * @author lazyman
  */
 public abstract class BaseSearchDataProvider<C extends Serializable, T extends Serializable>
-        extends BaseSortableDataProvider<T> {
+        extends BaseSortableDataProvider<T>
+        implements IterativeExportSupport<T> {
 
     private final IModel<Search<C>> search;
 
@@ -126,4 +129,25 @@ public void detach() {
         super.detach();
         search.detach();
     }
+
+    /**
+     * Default implementation throws UnsupportedOperationException.
+     * Subclasses should override this method to support streaming CSV export.
+     */
+    @Override
+    public void exportIterative(
+            ObjectHandler<T> handler,
+            Task task,
+            OperationResult result) throws CommonException {
+        throw new UnsupportedOperationException("Subclass must override exportIterative");
+    }
+
+    /**
+     * Returns false by default. Subclasses that implement exportIterative()
+     * should override this to return true.
+     */
+    @Override
+    public boolean supportsIterativeExport() {
+        return false;
+    }
 }

gui/admin-gui/src/main/java/com/evolveum/midpoint/gui/impl/component/data/provider/ContainerListDataProvider.java

@@ -20,6 +20,7 @@
 import com.evolveum.midpoint.gui.api.factory.wrapper.PrismContainerWrapperFactory;
 import com.evolveum.midpoint.gui.api.factory.wrapper.WrapperContext;
 import com.evolveum.midpoint.gui.api.prism.wrapper.PrismContainerValueWrapper;
+import com.evolveum.midpoint.gui.impl.prism.wrapper.PrismContainerValueWrapperImpl;
 import com.evolveum.midpoint.gui.api.util.WebComponentUtil;
 import com.evolveum.midpoint.gui.api.util.WebModelServiceUtils;
 import com.evolveum.midpoint.gui.impl.component.search.Search;
@@ -30,7 +31,10 @@
 import com.evolveum.midpoint.schema.SelectorOptions;
 import com.evolveum.midpoint.schema.result.OperationResult;
 import com.evolveum.midpoint.task.api.Task;
+import com.evolveum.midpoint.schema.ObjectHandler;
+import com.evolveum.midpoint.util.exception.CommonException;
 import com.evolveum.midpoint.util.exception.SchemaException;
+import com.evolveum.midpoint.util.exception.SystemException;
 import com.evolveum.midpoint.util.logging.LoggingUtils;
 import com.evolveum.midpoint.util.logging.Trace;
 import com.evolveum.midpoint.util.logging.TraceManager;
@@ -117,6 +121,15 @@ protected PrismContainerValueWrapper<C> createWrapper(C object, Task task, Opera
         return (PrismContainerValueWrapper<C>) factory.createValueWrapper(null, object.asPrismContainerValue(), ValueStatus.NOT_CHANGED, context);
     }
 
+    /**
+     * Creates a lightweight wrapper for export purposes.
+     * This skips child wrapper creation which is the main performance bottleneck.
+     * The wrapper only holds the PrismContainerValue - columns access data via getRealValue().
+     */
+    protected PrismContainerValueWrapper<C> createExportWrapper(C object) {
+        return new PrismContainerValueWrapperImpl<>(null, object.asPrismContainerValue(), ValueStatus.NOT_CHANGED);
+    }
+
     @Override
     protected int internalSize() {
         LOGGER.trace("begin::internalSize()");
@@ -146,4 +159,53 @@ public void detach() {
         super.detach();
         getAvailableData().clear();
     }
+
+    @Override
+    public boolean supportsIterativeExport() {
+        return true;
+    }
+
+    /**
+     * Streaming export using JDBC cursor-based streaming.
+     * This method does not load all data into memory - uses true JDBC streaming.
+     * Streaming is enabled by setting iterationPageSize to -1.
+     * Uses lightweight wrapper to skip expensive child wrapper creation.
+     */
+    @Override
+    public void exportIterative(
+            ObjectHandler<PrismContainerValueWrapper<C>> handler,
+            Task task,
+            OperationResult result) throws CommonException {
+
+        ObjectQuery query = getQuery();
+        if (query == null) {
+            query = getPrismContext().queryFactory().createQuery();
+        }
+        // Set ordering from current sort settings (no offset/limit for full export)
+        query.setPaging(createPaging(0, Integer.MAX_VALUE));
+
+        if (LOGGER.isTraceEnabled()) {
+            LOGGER.trace("exportIterative: Query {} with {}", getType().getSimpleName(), query.debugDump());
+        }
+
+        // Enable JDBC streaming mode by setting iterationPageSize to -1
+        Collection<SelectorOptions<GetOperationOptions>> streamingOptions =
+                SelectorOptions.updateRootOptions(options,
+                        opt -> opt.setIterationPageSize(-1), GetOperationOptions::new);
+
+        getModelService().searchContainersIterative(
+                getType(),
+                query,
+                (object, opResult) -> {
+                    PrismContainerValueWrapper<C> wrapper = createExportWrapper(object);
+                    if (wrapper != null) {
+                        return handler.handle(wrapper, opResult);
+                    }
+                    return true;
+                },
+                streamingOptions,
+                task,
+                result
+        );
+    }
 }

gui/admin-gui/src/main/java/com/evolveum/midpoint/gui/impl/component/data/provider/IterativeExportSupport.java

@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) 2010-2026 Evolveum and contributors
+ *
+ * Licensed under the EUPL-1.2 or later.
+ */
+
+package com.evolveum.midpoint.gui.impl.component.data.provider;
+
+import com.evolveum.midpoint.schema.ObjectHandler;
+import com.evolveum.midpoint.schema.result.OperationResult;
+import com.evolveum.midpoint.task.api.Task;
+import com.evolveum.midpoint.util.exception.CommonException;
+
+/**
+ * Interface for DataProviders that support iterative export.
+ * This allows streaming export without loading all data into memory.
+ *
+ * @param <T> The type of items being exported
+ */
+public interface IterativeExportSupport<T> {
+
+    /**
+     * Execute iterative search and pass each item to the handler.
+     * The search will stop if the handler returns false.
+     *
+     * @param handler Handler to process each item. Returns true to continue, false to stop.
+     * @param task Task for the operation
+     * @param result Operation result
+     * @throws CommonException if an error occurs during the search
+     */
+    void exportIterative(ObjectHandler<T> handler, Task task, OperationResult result) throws CommonException;
+
+    /**
+     * Returns true if this provider actually supports iterative export.
+     * Default is true, but BaseSearchDataProvider overrides to return false
+     * so that subclasses must explicitly enable support.
+     */
+    default boolean supportsIterativeExport() {
+        return true;
+    }
+}

gui/admin-gui/src/main/java/com/evolveum/midpoint/gui/impl/component/data/provider/SelectableBeanContainerDataProvider.java

@@ -15,6 +15,10 @@
 import com.evolveum.midpoint.gui.api.util.WebComponentUtil;
 import com.evolveum.midpoint.gui.api.util.WebModelServiceUtils;
 
+import com.evolveum.midpoint.prism.PrismContext;
+import com.evolveum.midpoint.schema.ObjectHandler;
+import com.evolveum.midpoint.web.component.util.SelectableBean;
+
 import org.apache.wicket.Component;
 import org.apache.wicket.model.IModel;
 import org.jetbrains.annotations.NotNull;
@@ -71,4 +75,51 @@ protected void addCachedSize(Map<Serializable, CachedSize> cache, CachedSize new
     protected boolean match(C selectedValue, C foundValue) {
         return selectedValue.asPrismContainerValue().equivalent(foundValue.asPrismContainerValue());
     }
+
+    @Override
+    public boolean supportsIterativeExport() {
+        return true;
+    }
+
+    /**
+     * Streaming export using JDBC cursor-based streaming.
+     * This method does not load all data into memory - uses true JDBC streaming.
+     * Streaming is enabled by setting iterationPageSize to -1.
+     */
+    @Override
+    public void exportIterative(
+            ObjectHandler<SelectableBean<C>> handler,
+            Task task,
+            OperationResult result) throws CommonException {
+
+        ObjectQuery query = getQuery();
+        if (query == null) {
+            query = PrismContext.get().queryFactory().createQuery();
+        }
+        // Set ordering from current sort settings (no offset/limit for full export)
+        query.setPaging(createPaging(0, Integer.MAX_VALUE));
+
+        // Enable JDBC streaming mode by setting iterationPageSize to -1
+        Collection<SelectorOptions<GetOperationOptions>> streamingOptions =
+                SelectorOptions.updateRootOptions(getSearchOptions(),
+                        opt -> opt.setIterationPageSize(-1), GetOperationOptions::new);
+
+        searchObjectsIterative(getType(), query,
+                (object, opResult) -> {
+                    SelectableBean<C> wrapper = createDataObjectWrapper(object);
+                    return handler.handle(wrapper, opResult);
+                },
+                streamingOptions, task, result);
+    }
+
+    /**
+     * Override this method to use a different iterative search implementation.
+     * Default implementation uses ModelService.searchContainersIterative().
+     */
+    protected void searchObjectsIterative(Class<C> type, ObjectQuery query,
+            ObjectHandler<C> handler,
+            Collection<SelectorOptions<GetOperationOptions>> options,
+            Task task, OperationResult result) throws CommonException {
+        getModelService().searchContainersIterative(type, query, handler, options, task, result);
+    }
 }

gui/admin-gui/src/main/java/com/evolveum/midpoint/gui/impl/component/data/provider/SelectableBeanObjectDataProvider.java

@@ -25,6 +25,7 @@
 import com.evolveum.midpoint.prism.PrismObject;
 import com.evolveum.midpoint.prism.query.ObjectQuery;
 import com.evolveum.midpoint.schema.GetOperationOptions;
+import com.evolveum.midpoint.schema.ObjectHandler;
 import com.evolveum.midpoint.schema.SelectorOptions;
 import com.evolveum.midpoint.schema.result.OperationResult;
 import com.evolveum.midpoint.task.api.Task;
@@ -129,4 +130,50 @@ public ObjectPaging createPaging(long offset, long pageSize) {
     public void setTaskConsumer(Consumer<Task> taskConsumer) {
         this.taskConsumer = taskConsumer;
     }
+
+    @Override
+    public boolean supportsIterativeExport() {
+        return true;
+    }
+
+    /**
+     * Streaming export using searchObjectsIterative with JDBC streaming.
+     * This method does not load all data into memory - uses true JDBC streaming.
+     * Streaming is enabled by setting iterationPageSize to -1.
+     */
+    @Override
+    public void exportIterative(
+            ObjectHandler<SelectableBean<O>> handler,
+            Task task,
+            OperationResult result) throws CommonException {
+
+        ObjectQuery query = getQuery();
+        if (query == null) {
+            query = getPrismContext().queryFactory().createQuery();
+        }
+        // Set ordering from current sort settings (no offset/limit for full export)
+        query.setPaging(createPaging(0, Integer.MAX_VALUE));
+
+        if (LOGGER.isTraceEnabled()) {
+            LOGGER.trace("exportIterative: Query {} with {}", getType().getSimpleName(), query.debugDump());
+        }
+
+        // Enable JDBC streaming mode by setting iterationPageSize to -1
+        Collection<SelectorOptions<GetOperationOptions>> streamingOptions =
+                SelectorOptions.updateRootOptions(getSearchOptions(),
+                        opt -> opt.setIterationPageSize(-1), GetOperationOptions::new);
+
+        getModelService().searchObjectsIterative(
+                getType(),
+                query,
+                (object, opResult) -> {
+                    O objectable = object.asObjectable();
+                    SelectableBean<O> wrapper = createDataObjectWrapper(objectable);
+                    return handler.handle(wrapper, opResult);
+                },
+                streamingOptions,
+                task,
+                result
+        );
+    }
 }