diff --git a/api/src/org/labkey/api/data/FileSqlScriptProvider.java b/api/src/org/labkey/api/data/FileSqlScriptProvider.java index 016e5c0a61b..43c42fd904a 100644 --- a/api/src/org/labkey/api/data/FileSqlScriptProvider.java +++ b/api/src/org/labkey/api/data/FileSqlScriptProvider.java @@ -18,7 +18,6 @@ import org.apache.commons.collections4.ComparatorUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -34,6 +33,7 @@ import org.labkey.api.util.FileUtil; import org.labkey.api.util.PageFlowUtil; import org.labkey.api.util.Path; +import org.labkey.api.util.logging.LogHelper; import org.labkey.api.vcs.Vcs; import org.labkey.api.vcs.VcsService; import org.labkey.api.view.JspTemplate; @@ -57,14 +57,9 @@ import static org.apache.commons.lang3.StringUtils.isBlank; -/** - * User: adam - * Date: Sep 18, 2007 - * Time: 10:26:29 AM - */ public class FileSqlScriptProvider implements SqlScriptProvider { - private static final Logger _log = LogManager.getLogger(FileSqlScriptProvider.class); + private static final Logger _log = LogHelper.getLogger(FileSqlScriptProvider.class, "SQL script issues"); private final Module _module; @@ -237,16 +232,14 @@ public String getProviderName() public void saveScript(DbSchema schema, String description, String contents) throws IOException { - saveScript(schema, description, contents, false); + saveScript(description, contents, getScriptDirectory(schema.getSqlDialect()), false); } - public void saveScript(DbSchema schema, String description, String contents, boolean overwrite) throws IOException + public void saveScript(String description, String contents, File scriptsDir, boolean overwrite) throws IOException { if (!AppProps.getInstance().isDevMode()) throw new IllegalStateException("Can't save scripts while in production mode"); - File scriptsDir = getScriptDirectory(schema.getSqlDialect()); - if (!scriptsDir.exists()) throw new IllegalStateException("SQL scripts directory not found"); diff --git a/api/src/org/labkey/api/mcp/McpContext.java b/api/src/org/labkey/api/mcp/McpContext.java index d1b0975f46f..6078214d965 100644 --- a/api/src/org/labkey/api/mcp/McpContext.java +++ b/api/src/org/labkey/api/mcp/McpContext.java @@ -4,15 +4,16 @@ import org.labkey.api.data.Container; import org.labkey.api.security.User; import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.util.QuietCloser; import org.labkey.api.view.UnauthorizedException; import org.labkey.api.writer.ContainerUser; import org.springframework.ai.chat.model.ToolContext; import java.util.Map; /** - * TODO MCP tool calling supports passing along a ToolContext. And most all - * interesting tools probably need a User and Container. This is not all hooked-up - * yet. This is an area for further investiation. + * TODO MCP tool calling supports passing along a ToolContext. And most all + * interesting tools probably need a User and Container. This is not all hooked-up + * yet. This is an area for further investigation. */ public class McpContext implements ContainerUser { @@ -67,17 +68,17 @@ public User getUser() return ret; } - public static AutoCloseable withContext(ContainerUser ctx) + public static QuietCloser withContext(ContainerUser ctx) { return with(new McpContext(ctx)); } - public static AutoCloseable withContext(Container container, User user) + public static QuietCloser withContext(Container container, User user) { return with(new McpContext(container, user)); } - private static AutoCloseable with(McpContext ctx) + private static QuietCloser with(McpContext ctx) { final McpContext prev = contexts.get(); contexts.set(ctx); diff --git a/api/src/org/labkey/api/util/SqlUtil.java b/api/src/org/labkey/api/util/SqlUtil.java new file mode 100644 index 00000000000..5ebfd6df848 --- /dev/null +++ b/api/src/org/labkey/api/util/SqlUtil.java @@ -0,0 +1,22 @@ +package org.labkey.api.util; + +public class SqlUtil +{ + public static String extractSql(String text) + { + if (text.startsWith("SELECT ")) + return text; + if (text.startsWith("WITH ") && text.contains("SELECT ")) + return text; + if (text.startsWith("PARAMETERS ") && text.contains("SELECT ")) + return text; + var sql = text.indexOf("```sql\n"); + if (sql >= 0) + { + var end = text.indexOf("```", sql+7); + if (end >= 0) + return text.substring(sql+7,end); + } + return null; + } +} diff --git a/core/src/org/labkey/core/admin/sql/ScriptReorderer.java b/core/src/org/labkey/core/admin/sql/ScriptReorderer.java index 7e93d786ef8..c2f2b963ea3 100644 --- a/core/src/org/labkey/core/admin/sql/ScriptReorderer.java +++ b/core/src/org/labkey/core/admin/sql/ScriptReorderer.java @@ -16,7 +16,7 @@ package org.labkey.core.admin.sql; -import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; import org.jetbrains.annotations.Nullable; import org.labkey.api.data.DbSchema; import org.labkey.api.util.PageFlowUtil; @@ -33,7 +33,6 @@ public class ScriptReorderer { public static final String COMMENT_REGEX = "((/\\*.+?\\*/)|(^[ \\t]*--.*?$))\\s*"; // Single-line or block comment, followed by white space - private static final String SCHEMA_NAME_REGEX = "((\\w+)\\.)?"; private final List>> _statementLists = new LinkedList<>(); private final List _endingStatements = new LinkedList<>(); @@ -41,10 +40,12 @@ public class ScriptReorderer private Map> _currentStatements; private final DbSchema _schema; + private final String SCHEMA_NAME_REGEX; private final String TABLE_NAME_REGEX; private final String TABLE_NAME2_REGEX; private final String TABLE_NAME_NO_UNDERSCORE_REGEX; private final String STATEMENT_ENDING_REGEX; + private final String CONSTRAINT_NAME_REGEX; private String _contents; private int _row = 0; @@ -56,15 +57,19 @@ public class ScriptReorderer if (_schema.getSqlDialect().isSqlServer()) { - TABLE_NAME_REGEX = "(?" + SCHEMA_NAME_REGEX + "(#?\\w+))"; // # allows for temp table names + SCHEMA_NAME_REGEX = "(((\\w+)|(\\[\\w+\\]))\\.)?"; // optional [] around schema name + TABLE_NAME_REGEX = "(?
" + SCHEMA_NAME_REGEX + "((#?\\w+)|(\\[#?\\w+\\])))"; // # allows for temp table names, optional [] around table name TABLE_NAME_NO_UNDERSCORE_REGEX = null; STATEMENT_ENDING_REGEX = "((; GO\\s*$)|(;\\s*$)|( GO\\s*$))\\s*"; // Semicolon, GO, or both + CONSTRAINT_NAME_REGEX = "((\\w+)|(\\[\\w+\\]))"; // optional [] around name } else { + SCHEMA_NAME_REGEX = "((\\w+)\\.)?"; TABLE_NAME_REGEX = "(?
" + SCHEMA_NAME_REGEX + "(\\w+))"; TABLE_NAME_NO_UNDERSCORE_REGEX = "(?
" + SCHEMA_NAME_REGEX + "([[a-zA-Z0-9]]+))"; STATEMENT_ENDING_REGEX = ";(\\s*?)((--)[^\\n]*)?$(\\s*)"; + CONSTRAINT_NAME_REGEX = "(\\w+)"; } TABLE_NAME2_REGEX = TABLE_NAME_REGEX.replace("table", "table2"); @@ -89,7 +94,7 @@ public String getReorderedScript(boolean isHtml) patterns.add(new SqlPattern(getRegExWithPrefix("UPDATE (ON )?"), Type.Table, Operation.AlterRows)); patterns.add(new SqlPattern(getRegExWithPrefix("DELETE FROM "), Type.Table, Operation.AlterRows)); - patterns.add(new SqlPattern("CREATE (UNIQUE )?((NON)?CLUSTERED )?INDEX (IF NOT EXISTS )?\\w+? ON " + TABLE_NAME_REGEX + ".+?" + STATEMENT_ENDING_REGEX, Type.Table, Operation.Other)); + patterns.add(new SqlPattern("CREATE (UNIQUE )?((NON)?CLUSTERED )?INDEX (IF NOT EXISTS )?\\[?(\\w+?)\\]? ON " + TABLE_NAME_REGEX + ".+?" + STATEMENT_ENDING_REGEX, Type.Table, Operation.Other)); patterns.add(new SqlPattern(getRegExWithPrefix("CREATE TABLE "), Type.Table, Operation.Other)); patterns.add(new SqlPattern(getRegExWithPrefix("TRUNCATE( TABLE)? "), Type.Table, Operation.Other)); @@ -104,7 +109,7 @@ public String getReorderedScript(boolean isHtml) // All other sp_renames patterns.add(new SqlPattern("(EXEC(UTE)? )?sp_rename (@objname\\s*=\\s*)?'" + TABLE_NAME_REGEX + ".*?'.+?" + STATEMENT_ENDING_REGEX, Type.Table, Operation.Other)); - patterns.add(new SqlPattern("EXEC(UTE)? core\\.fn_dropifexists\\s*'(?
\\w+)'\\s*,\\s*'(?\\w+)'\\s*,\\s*'(TABLE|COLUMN|INDEX|DEFAULT|CONSTRAINT)'.*?" + STATEMENT_ENDING_REGEX, Type.Table, Operation.Other)); + patterns.add(new SqlPattern("EXEC(UTE)? core\\.fn_dropifexists\\s*(@objname\\s*=\\s*)?'(?
\\w+)'\\s*,\\s*(@objschema\\s*=\\s*)?'(?\\w+)'\\s*,\\s*(@objtype\\s*=\\s*)?'(TABLE|COLUMN|INDEX|DEFAULT|CONSTRAINT)'.*?" + STATEMENT_ENDING_REGEX, Type.Table, Operation.Other)); patterns.add(new SqlPattern("EXEC(UTE)? core\\.fn_dropifexists\\s*'(\\w+)'\\s*,\\s*'(?\\w+)'.*?" + STATEMENT_ENDING_REGEX, Type.NonTable, Operation.Other)); // DROP INDEX on SQL Server follows a similar pattern to CREATE INDEX (above) @@ -134,7 +139,7 @@ public String getReorderedScript(boolean isHtml) patterns.add(new SqlPattern("DO (\\S+) (.+?) END \\1" + STATEMENT_ENDING_REGEX, Type.NonTable, Operation.Other)); } - patterns.add(new SqlPattern("ALTER TABLE " + TABLE_NAME_REGEX + " ADD CONSTRAINT \\w+ FOREIGN KEY \\([^\\)]+?\\) REFERENCES " + TABLE_NAME2_REGEX + " \\([^\\)]+?\\).*?" + STATEMENT_ENDING_REGEX, Type.Table, Operation.Other)); + patterns.add(new SqlPattern("ALTER TABLE " + TABLE_NAME_REGEX + " (WITH CHECK )?ADD CONSTRAINT " + CONSTRAINT_NAME_REGEX + " FOREIGN KEY\\s*\\([^\\)]+?\\) REFERENCES " + TABLE_NAME2_REGEX + " \\([^\\)]+?\\).*?" + STATEMENT_ENDING_REGEX, Type.Table, Operation.Other)); // Put this at the end to capture all other ALTER TABLE statements (i.e., not RENAMEs) patterns.add(new SqlPattern(getRegExWithPrefix("ALTER TABLE (IF EXISTS )?(ONLY )?"), Type.Table, Operation.Other)); @@ -320,7 +325,7 @@ private void addStatement(String tableName, @Nullable String tableName2, String if (null != tableName2 && index(tableName2) > index(tableName)) tableName = tableName2; - String key = tableName.toLowerCase(); + String key = tableName.replace("[", "").replace("]", "").toLowerCase(); Collection tableStatements = _currentStatements.computeIfAbsent(key, k -> new LinkedList<>()); @@ -377,14 +382,14 @@ private void appendStatement(StringBuilder sb, Statement statement) if (null != tableName) { String schemaName = null; - boolean containsTableName = StringUtils.containsIgnoreCase(sql, tableName); + boolean containsTableName = Strings.CI.contains(sql, tableName); if (!containsTableName && tableName.contains(".")) { String[] parts = tableName.split("\\."); tableName = parts[0]; schemaName = parts[1]; - containsTableName = StringUtils.containsIgnoreCase(sql, tableName); + containsTableName = Strings.CI.contains(sql, tableName); } if (containsTableName) diff --git a/core/src/org/labkey/core/admin/sql/SqlScriptController.java b/core/src/org/labkey/core/admin/sql/SqlScriptController.java index 75f30ccfa17..73f4e5678a5 100644 --- a/core/src/org/labkey/core/admin/sql/SqlScriptController.java +++ b/core/src/org/labkey/core/admin/sql/SqlScriptController.java @@ -16,7 +16,7 @@ package org.labkey.core.admin.sql; -import org.apache.logging.log4j.LogManager; +import jakarta.servlet.http.HttpSession; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -25,6 +25,7 @@ import org.labkey.api.Constants; import org.labkey.api.action.ApiResponse; import org.labkey.api.action.ApiSimpleResponse; +import org.labkey.api.action.FormHandlerAction; import org.labkey.api.action.FormViewAction; import org.labkey.api.action.IgnoresAllocationTracking; import org.labkey.api.action.ReadOnlyApiAction; @@ -32,11 +33,11 @@ import org.labkey.api.action.SpringActionController; import org.labkey.api.admin.AdminUrls; import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; import org.labkey.api.data.CoreSchema; import org.labkey.api.data.DbSchema; import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.DbScope; import org.labkey.api.data.FileSqlScriptProvider; import org.labkey.api.data.SqlScriptManager; import org.labkey.api.data.SqlScriptRunner; @@ -44,25 +45,35 @@ import org.labkey.api.data.SqlScriptRunner.SqlScriptProvider; import org.labkey.api.data.UpgradeCode; import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.mcp.McpContext; +import org.labkey.api.mcp.McpService; +import org.labkey.api.mcp.McpService.MessageResponse; import org.labkey.api.module.AllowedDuringUpgrade; import org.labkey.api.module.Module; import org.labkey.api.module.ModuleContext; import org.labkey.api.module.ModuleLoader; import org.labkey.api.security.AdminConsoleAction; +import org.labkey.api.security.Crypt; import org.labkey.api.security.RequiresPermission; import org.labkey.api.security.User; import org.labkey.api.security.permissions.AbstractActionPermissionTest; import org.labkey.api.security.permissions.AdminOperationsPermission; import org.labkey.api.security.permissions.TroubleshooterPermission; import org.labkey.api.settings.AppProps; +import org.labkey.api.util.ButtonBuilder; +import org.labkey.api.util.CsrfInput; +import org.labkey.api.util.FileUtil; import org.labkey.api.util.HtmlString; import org.labkey.api.util.HtmlStringBuilder; import org.labkey.api.util.LinkBuilder; import org.labkey.api.util.PageFlowUtil; import org.labkey.api.util.Pair; +import org.labkey.api.util.Path; +import org.labkey.api.util.QuietCloser; +import org.labkey.api.util.SqlUtil; import org.labkey.api.util.TestContext; import org.labkey.api.util.URLHelper; -import org.labkey.api.util.CsrfInput; +import org.labkey.api.util.logging.LogHelper; import org.labkey.api.vcs.Vcs; import org.labkey.api.vcs.VcsService; import org.labkey.api.view.ActionURL; @@ -72,9 +83,11 @@ import org.labkey.api.view.NavTree; import org.labkey.api.view.RedirectException; import org.labkey.core.admin.AdminController; +import org.springframework.ai.chat.client.ChatClient; import org.springframework.validation.BindException; import org.springframework.validation.Errors; import org.springframework.web.servlet.ModelAndView; +import org.springframework.web.servlet.mvc.Controller; import java.io.File; import java.io.IOException; @@ -99,7 +112,7 @@ public class SqlScriptController extends SpringActionController { private static final DefaultActionResolver _actionResolver = new DefaultActionResolver(SqlScriptController.class); - private static final Logger LOG = LogManager.getLogger(SqlScriptController.class); + private static final Logger LOG = LogHelper.getLogger(SqlScriptController.class, "Execution of upgrade code"); public SqlScriptController() { @@ -344,7 +357,6 @@ public void addNavTrail(NavTree root) // We sort the list of scripts, so insist on an ArrayList private void appendScripts(HtmlStringBuilder html, ArrayList scripts) { - Container c = getContainer(); html.unsafeAppend("
\n"); if (scripts.isEmpty()) @@ -357,16 +369,10 @@ private void appendScripts(HtmlStringBuilder html, ArrayList scripts) for (SqlScript script : scripts) { - ActionURL url = new ActionURL(ScriptAction.class, c); - url.addParameter("moduleName", script.getProvider().getProviderName()); - url.addParameter("filename", script.getDescription()); - - html.unsafeAppend(""); - html.append(script.getDescription()); - - html.unsafeAppend("
\n"); + ActionURL url = getScriptURL(ScriptAction.class, script); + html.append(new LinkBuilder(script.getDescription()).href(url).clearClasses()) + .append(HtmlString.BR) + .append("\n"); } } @@ -470,7 +476,7 @@ private static List getConsolidators(double fromVersion, dou else if (includeSingleScripts) { String filename = consolidator.getFilename(); - SqlScript script = scripts.get(0); + SqlScript script = scripts.getFirst(); // Skip if the single script in this range is the consolidation script if (!script.getDescription().equalsIgnoreCase(filename) && 0.0 != script.getFromVersion()) @@ -596,7 +602,6 @@ private static class ScriptConsolidator private final List _scripts; private final double _targetFrom; private final double _targetTo; - private final double _actualTo; private Collection _errors = null; @@ -607,7 +612,6 @@ private ScriptConsolidator(FileSqlScriptProvider provider, DbSchema schema, doub _targetFrom = targetFrom; _targetTo = targetTo; _scripts = SqlScriptManager.get(provider, schema).getRecommendedScripts(provider.getScripts(schema), targetFrom, targetTo); - _actualTo = _scripts.isEmpty() ? -1 : _scripts.get(_scripts.size() - 1).getToVersion(); } private List getScripts() @@ -863,7 +867,7 @@ private ScriptConsolidator getConsolidator(ConsolidateForm form) return getConsolidator(provider, DbSchema.get(form.getSchema(), DbSchemaType.Module), form.getFromVersion(), form.getToVersion()); } - protected ScriptConsolidator getConsolidator(FileSqlScriptProvider provider, DbSchema schema, double fromVersion, double toVersion) + private ScriptConsolidator getConsolidator(FileSqlScriptProvider provider, DbSchema schema, double fromVersion, double toVersion) { return new ScriptConsolidator(provider, schema, fromVersion, toVersion); } @@ -1117,13 +1121,31 @@ protected void renderScript(SqlScript script, PrintWriter out) protected void renderButtons(SqlScript script, PrintWriter out) { - ActionURL url = new ActionURL(ReorderScriptAction.class, getViewContext().getContainer()); - url.addParameter("moduleName", script.getProvider().getProviderName()); - url.addParameter("filename", script.getDescription()); - out.println(PageFlowUtil.button("Reorder Script").href(url)); + out.println(PageFlowUtil.button("Reorder Script").href(getScriptURL(ReorderScriptAction.class, script))); + if (McpService.get().isReady()) + { + out.println( + PageFlowUtil.button("Clean Up Script") + .href(getScriptURL(CleanUpScriptAction.class, script)) + .tooltip("Remove redundant and unnecessary statements. This uses AI, so it may take some time and its results must be carefully reviewed.") + ); + SqlDialect dialect = script.getSchema().getSqlDialect(); + String theOther = getTheOtherDialectDescription(script.getSchema().getSqlDialect()); + out.println( + PageFlowUtil.button("Migrate to " + theOther) + .href(getScriptURL(MigrateScriptAction.class, script)) + .tooltip("Migrate this " + dialect.getProductName() + " SQL script to " + theOther + " syntax. This uses AI, so it may take some time and its results must be carefully reviewed.") + ); + } } } + private static ActionURL getScriptURL(Class actionClass, SqlScript script) + { + return new ActionURL(actionClass, ContainerManager.getRoot()) + .addParameter("moduleName", script.getProvider().getProviderName()) + .addParameter("filename", script.getDescription()); + } @RequiresPermission(AdminOperationsPermission.class) public class ReorderScriptAction extends ScriptAction @@ -1141,6 +1163,29 @@ protected String getActionDescription() } } + private static class ReorderingScriptView extends ScriptView + { + private ReorderingScriptView(SqlScript script) + { + super(script); + } + + @Override + protected void renderScript(SqlScript script, PrintWriter out) + { + out.println(""); + ScriptReorderer reorderer = new ScriptReorderer(script.getSchema(), script.getContents()); + out.println(reorderer.getReorderedScript(true)); + out.println("
"); + } + + @Override + protected void renderButtons(SqlScript script, PrintWriter out) + { + out.println(PageFlowUtil.button("Save Reordered Script to " + script.getDescription()).href(getScriptURL(SaveReorderedScriptAction.class, script))); + out.println(PageFlowUtil.button("Back").href(getScriptURL(ScriptAction.class, script))); + } + } @RequiresPermission(AdminOperationsPermission.class) public static class ReorderAllScriptsAction extends SimpleViewAction @@ -1148,7 +1193,7 @@ public static class ReorderAllScriptsAction extends SimpleViewAction @Override public ModelAndView getView(Object o, BindException errors) { - return new HttpView() + return new HttpView<>() { @Override protected void renderInternal(Object model, PrintWriter out) @@ -1180,7 +1225,6 @@ public void addNavTrail(NavTree root) } } - @RequiresPermission(AdminOperationsPermission.class) public class SaveReorderedScriptAction extends ScriptAction { @@ -1189,48 +1233,207 @@ protected ModelAndView getScriptView(SqlScript script) throws RedirectException, { ScriptReorderer reorderer = new ScriptReorderer(script.getSchema(), script.getContents()); String reorderedScript = reorderer.getReorderedScript(false); - ((FileSqlScriptProvider)script.getProvider()).saveScript(script.getSchema(), script.getDescription(), reorderedScript, true); - - final ActionURL url = new ActionURL(ScriptAction.class, getContainer()); - url.addParameter("moduleName", script.getProvider().getProviderName()); - url.addParameter("filename", script.getDescription()); + throw new RedirectException(saveScript(script, reorderedScript, null)); + } + } + + // Save the new script contents to the specified script file and return an ActionURL to show it + private ActionURL saveScript(SqlScript script, String newContents, @Nullable File scriptDir) throws IOException + { + FileSqlScriptProvider provider = (FileSqlScriptProvider) script.getProvider(); + if (scriptDir == null) + { + scriptDir = provider.getScriptDirectory(script.getSchema().getSqlDialect()); + } + else if (!scriptDir.exists()) + { + FileUtil.mkdir(scriptDir); + } + provider.saveScript(script.getDescription(), newContents, scriptDir, true); + return getScriptURL(ScriptAction.class, script); + } + + protected abstract class BaseAIScriptAction extends ScriptAction + { + @Override + protected ModelAndView getScriptView(SqlScript script) + { + McpService mcpService = McpService.get(); + if (mcpService.isReady()) + { + String prompt = getPrompt(DbScope.getLabKeyScope().getSqlDialect(), script); + HttpSession session = getViewContext().getSession(); + ChatClient client = mcpService.getChat(session, getChatName(), () -> prompt); + try (QuietCloser _ = McpContext.withContext(getViewContext())) + { + List responses = mcpService.sendMessageEx(client,"```sql\n" + script.getContents() + "```"); + McpService.get().close(session, client); + HtmlStringBuilder viewBuilder = HtmlStringBuilder.of(); + StringBuilder newContents = new StringBuilder(); + responses + .forEach(r -> { + viewBuilder.append(r.html()); + String sql = SqlUtil.extractSql(r.text()); + if (sql != null) + newContents.append(sql); + }); + viewBuilder.append(new ButtonBuilder("Save Updated Script to " + script.getDescription()).href(getSaveScriptActionURL(script, newContents.toString(), null)).usePost()); + return new HtmlView(viewBuilder); + } + } + return new HtmlView(HtmlString.of("McpService is not ready")); + } + + // Stashes the script contents in session and returns an ActionURL to the save action + protected ActionURL getSaveScriptActionURL(SqlScript script, String newContents, @Nullable File scriptsDir) + { + HttpSession session = getViewContext().getSession(); + String key = Crypt.SHA256.digest(script.getDescription() + newContents); + session.setAttribute(key, new ScriptToSave(script, newContents, scriptsDir)); - throw new RedirectException(url); + return new ActionURL(SaveScriptAction.class, ContainerManager.getRoot()).addParameter("key", key); } + + abstract protected String getPrompt(SqlDialect dialect, SqlScript script); + + abstract protected String getChatName(); } + private static final String CLEAN_UP_PROMPT = """ + Refactor the script to provide a clean, "final state" version, removing redundant and unnecessary statements. + + Note that the `core.fn_dropifexists` stored procedure is used to drop a TABLE, VIEW, COLUMN, or other database + object if it exists. In most cases, the first parameter specifies the table name, the second parameter specifies + the schema name, the third parameter specifies the object type, and the optional fourth parameter specifies + other details such as a column name. Here are some examples: + - `EXEC core.fn_dropifexists @objname = 'MyTable', @objschema = 'MySchema', @objtype = 'TABLE'` is the same as `DROP TABLE IF EXISTS MySchema.MyTable` + - `EXEC core.fn_dropifexists 'MyTable', 'MySchema', 'TABLE'` is the same as `DROP TABLE IF EXISTS MySchema.MyTable` + - `EXEC core.fn_dropifexists 'MyTable', 'MySchema', 'COLUMN', 'MyColumn` is the same as `ALTER TABLE TableName DROP COLUMN IF EXISTS ColumnName` + + Please do the following: + - Consolidate all iterative changes (column additions, PK changes, and renames) into the initial CREATE TABLE statements. + - Remove unnecessary DROP TABLE statements and core.fn_dropifexists calls, for example, those that come before a table has been created. + - Remove all intermediate DROP and ALTER statements that are superseded by later logic. + - Remove CREATE TABLE and ALTER TABLE statements followed by DROP TABLE or and core.fn_dropifexists 'TABLE' call on that same table. + + Include a summary of the changes you made at the end. + """; - private static class ReorderingScriptView extends ScriptView + @RequiresPermission(AdminOperationsPermission.class) + public class CleanUpScriptAction extends BaseAIScriptAction { - private ReorderingScriptView(SqlScript script) + @Override + protected String getPrompt(SqlDialect dialect, SqlScript script) { - super(script); + String youAre = "You are a " + dialect.getProductName() + (dialect.isSqlServer() ? " T-SQL" : " SQL") + " expert.\n"; + String yourTask = "Your task is to clean up this " + dialect.getProductName() + " SQL script" + (script.getFromVersion() == 0.0 ? ", which creates a brand new database schema and populates it with tables" : "") + ".\n"; + return youAre + yourTask + CLEAN_UP_PROMPT; } @Override - protected void renderScript(SqlScript script, PrintWriter out) + protected String getChatName() { - out.println(""); - ScriptReorderer reorderer = new ScriptReorderer(script.getSchema(), script.getContents()); - out.println(reorderer.getReorderedScript(true)); - out.println("
"); + return "SQL Script Cleaner"; } @Override - protected void renderButtons(SqlScript script, PrintWriter out) + protected String getActionDescription() + { + return "Clean Up " + super.getActionDescription(); + } + } + + private static String getTheOtherDialectDescription(SqlDialect dialect) + { + return dialect.isPostgreSQL() ? "Microsoft SQL Server" : "PostgreSQL"; + } + + private static String getTheOtherScriptDir(SqlDialect dialect) + { + return dialect.isPostgreSQL() ? "sqlserver" : "postgresql"; + } + + private static final String MIGRATE_TO_PG_PROMPT = """ + Include a summary of the changes you made at the end. + """; + + @RequiresPermission(AdminOperationsPermission.class) + public class MigrateScriptAction extends BaseAIScriptAction + { + @Override + protected String getPrompt(SqlDialect dialect, SqlScript script) { - ActionURL reorderUrl = new ActionURL(SaveReorderedScriptAction.class, getViewContext().getContainer()); - reorderUrl.addParameter("moduleName", script.getProvider().getProviderName()); - reorderUrl.addParameter("filename", script.getDescription()); - out.println(PageFlowUtil.button("Save Reordered Script to " + script.getDescription()).href(reorderUrl)); + String youAre = "You are an expert in Microsoft SQL Server T-SQL and PostgreSQL SQL.\n"; + String yourTask = "Given this " + dialect.getProductName() + " SQL script, create an equivalent SQL script that's compatible with " + getTheOtherDialectDescription(dialect) + ".\n"; + return youAre + yourTask + MIGRATE_TO_PG_PROMPT; + } - ActionURL backUrl = new ActionURL(ScriptAction.class, getViewContext().getContainer()); - backUrl.addParameter("moduleName", script.getProvider().getProviderName()); - backUrl.addParameter("filename", script.getDescription()); - out.println(PageFlowUtil.button("Back").href(backUrl)); + @Override + protected String getChatName() + { + return "SQL Script Migrator"; + } + + @Override + protected String getActionDescription() + { + return "Migrate " + super.getActionDescription(); + } + + @Override + protected ActionURL getSaveScriptActionURL(SqlScript script, String newContents, @Nullable File scriptDir) + { + SqlDialect dialect = script.getSchema().getSqlDialect(); + File dbscripts = ((FileSqlScriptProvider)script.getProvider()).getScriptDirectory(dialect).getParentFile(); + scriptDir = FileUtil.appendPath(dbscripts, Path.parse(getTheOtherScriptDir(dialect))); + return super.getSaveScriptActionURL(script, newContents, scriptDir); } } + private record ScriptToSave(SqlScript script, String contents, @Nullable File scriptsDir) {} + + public static class SaveScriptForm + { + private String _key; + + public String getKey() + { + return _key; + } + + public void setKey(String key) + { + _key = key; + } + } + + @RequiresPermission(AdminOperationsPermission.class) + public class SaveScriptAction extends FormHandlerAction + { + private ScriptToSave _sts; + private ActionURL _redirectURL; + + @Override + public void validateCommand(SaveScriptForm form, Errors errors) + { + _sts = (ScriptToSave)getViewContext().getSession().getAttribute(form.getKey()); + if (null == _sts) + errors.reject(ERROR_GENERIC, "Script not found"); + } + + @Override + public boolean handlePost(SaveScriptForm form, BindException errors) throws Exception + { + _redirectURL = saveScript(_sts.script(), _sts.contents(), _sts.scriptsDir()); + return true; + } + + @Override + public URLHelper getSuccessURL(SaveScriptForm saveScriptForm) + { + return _redirectURL; + } + } @RequiresPermission(AdminOperationsPermission.class) public class UnreachableScriptsAction extends SimpleViewAction @@ -1365,14 +1568,14 @@ public void validateCommand(UpgradeCodeForm form, Errors errors) Module module = ModuleLoader.getInstance().getModule(form.getModule()); if (null == module) { - errors.reject(ERROR_MSG, "Module not found"); + errors.reject(ERROR_GENERIC, "Module not found"); } else { _code = module.getUpgradeCode(); if (null == _code) { - errors.reject(ERROR_MSG, "Module doesn't have UpgradeCode"); + errors.reject(ERROR_GENERIC, "Module doesn't have UpgradeCode"); } else { @@ -1381,11 +1584,11 @@ public void validateCommand(UpgradeCodeForm form, Errors errors) _method = _code.getClass().getDeclaredMethod(form.getMethod(), ModuleContext.class); _ctx = ModuleLoader.getInstance().getModuleContextFromDatabase(form.getModule()); if (null == _ctx) - errors.reject(ERROR_MSG, "ModuleContext not found"); + errors.reject(ERROR_GENERIC, "ModuleContext not found"); } catch (NoSuchMethodException e) { - errors.reject(ERROR_MSG, "Method doesn't exist"); + errors.reject(ERROR_GENERIC, "Method doesn't exist"); } } } @@ -1400,7 +1603,7 @@ public ModelAndView getView(UpgradeCodeForm form, boolean reshow, BindException @Override public boolean handlePost(UpgradeCodeForm form, BindException errors) throws Exception { - LOG.info("Executing " + _method.getDeclaringClass().getSimpleName() + "." + _method.getName() + "(ModuleContext moduleContext)"); + LOG.info("Executing {}.{}(ModuleContext moduleContext)", _method.getDeclaringClass().getSimpleName(), _method.getName()); _method.invoke(_code, _ctx); return true; } diff --git a/core/src/org/labkey/core/mpc/McpServiceImpl.java b/core/src/org/labkey/core/mpc/McpServiceImpl.java index c79b8d64dd7..62939eebf62 100644 --- a/core/src/org/labkey/core/mpc/McpServiceImpl.java +++ b/core/src/org/labkey/core/mpc/McpServiceImpl.java @@ -497,7 +497,7 @@ public MessageResponse sendMessage(ChatClient chatSession, String message) // Spring AI GoogleGenAiChatModel bug: empty candidates cause NoSuchElementException // https://github.com/spring-projects/spring-ai/issues/4556 LOG.warn("Empty response from chat model (likely a filtered or empty candidate)", x); - return new MessageResponse("text/plain", "The model returned an empty response. Please try resubmitting and the problem continues, rephrase your question/prompt.", HtmlString.of("The model returned an empty response. Please try rephrasing your question.")); + return new MessageResponse("text/plain", "The model returned an empty response. Please try resubmitting and, if the problem continues, rephrase your question/prompt.", HtmlString.of("The model returned an empty response. Please try rephrasing your question.")); } } diff --git a/core/src/org/labkey/core/view/ShortURLFilter.java b/core/src/org/labkey/core/view/ShortURLFilter.java index 14996e36ee4..15f4124dfcb 100644 --- a/core/src/org/labkey/core/view/ShortURLFilter.java +++ b/core/src/org/labkey/core/view/ShortURLFilter.java @@ -31,17 +31,9 @@ /** * Looks for incoming URLs that match a short URL registered with ShortURLService. If so, the filter will bounce the * browser to the target URL. - * User: jeckels - * Date: 1/31/14 */ public class ShortURLFilter implements Filter { - @Override - public void destroy() - { - - } - @Override public void doFilter(ServletRequest req, ServletResponse resp, FilterChain chain) throws IOException, ServletException { diff --git a/query/src/org/labkey/query/controllers/QueryController.java b/query/src/org/labkey/query/controllers/QueryController.java index 5a3c88f2edc..15cf1f9298e 100644 --- a/query/src/org/labkey/query/controllers/QueryController.java +++ b/query/src/org/labkey/query/controllers/QueryController.java @@ -246,6 +246,7 @@ import org.labkey.api.util.Pair; import org.labkey.api.util.ResponseHelper; import org.labkey.api.util.ReturnURLString; +import org.labkey.api.util.SqlUtil; import org.labkey.api.util.StringExpression; import org.labkey.api.util.StringUtilsLabKey; import org.labkey.api.util.TestContext; @@ -8975,7 +8976,7 @@ static SqlResponse extractSql(List responses) if (null == sql) { var text = response.text(); - String sqlFind = extractSql(text); + String sqlFind = SqlUtil.extractSql(text); if (null != sqlFind) { sql = sqlFind; @@ -8987,22 +8988,4 @@ static SqlResponse extractSql(List responses) } return new SqlResponse(html.getHtmlString(), sql); } - - static String extractSql(String text) - { - if (text.startsWith("SELECT ")) - return text; - if (text.startsWith("WITH ") && text.contains("SELECT ")) - return text; - if (text.startsWith("PARAMETERS ") && text.contains("SELECT ")) - return text; - var sql = text.indexOf("```sql\n"); - if (sql >= 0) - { - var end = text.indexOf("```", sql+7); - if (end >= 0) - return text.substring(sql+7,end); - } - return null; - } }