From 8c55333d1d48099493fc06e74132c253ebe0a6dc Mon Sep 17 00:00:00 2001 From: Mayank Jha Date: Fri, 20 Feb 2026 15:59:04 -0800 Subject: [PATCH 1/2] fix: make simulation.json resource name comparison case-insensitive Co-Authored-By: Claude Opus 4.6 --- src/uipath/_cli/_evals/mocks/mocks.py | 5 ++-- tests/cli/test_debug_simulation.py | 34 +++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/src/uipath/_cli/_evals/mocks/mocks.py b/src/uipath/_cli/_evals/mocks/mocks.py index ed8f1c3c6..793ddeb19 100644 --- a/src/uipath/_cli/_evals/mocks/mocks.py +++ b/src/uipath/_cli/_evals/mocks/mocks.py @@ -73,11 +73,12 @@ def clear_execution_context() -> None: def _normalize_tool_name(name: str) -> str: - """Normalize tool name by replacing underscores with spaces. + """Normalize tool name by replacing underscores with spaces and lowering case. Tool names may use spaces in configuration but underscores in execution. + Comparison is case-insensitive. """ - return name.replace("_", " ") + return name.replace("_", " ").lower() def is_tool_simulated(tool_name: str) -> bool: diff --git a/tests/cli/test_debug_simulation.py b/tests/cli/test_debug_simulation.py index 61498ee19..12f561480 100644 --- a/tests/cli/test_debug_simulation.py +++ b/tests/cli/test_debug_simulation.py @@ -524,3 +524,37 @@ def test_handles_tool_name_normalization(self, temp_dir: str): assert is_tool_simulated("Web Search") is True clear_execution_context() + + def test_handles_tool_name_case_insensitive(self, temp_dir: str): + """Test that tool name comparison is case-insensitive.""" + config = { + "enabled": True, + "instructions": "Test", + "toolsToSimulate": [ + {"name": "Web Reader"}, + {"name": "Web Search"}, + ], + } + simulation_path = Path(temp_dir) / "simulation.json" + with open(simulation_path, "w", encoding="utf-8") as f: + json.dump(config, f) + + with patch("uipath._cli.cli_debug.Path.cwd", return_value=Path(temp_dir)): + mocking_ctx = load_simulation_config() + assert mocking_ctx is not None + + from uipath._cli._evals._span_collection import ExecutionSpanCollector + from uipath._cli._evals.mocks.mocks import set_execution_context + + span_collector = ExecutionSpanCollector() + set_execution_context(mocking_ctx, span_collector, "test-id") + + # Case-insensitive matching should work + assert is_tool_simulated("web reader") is True + assert is_tool_simulated("WEB READER") is True + assert is_tool_simulated("Web reader") is True + assert is_tool_simulated("web search") is True + assert is_tool_simulated("WEB SEARCH") is True + assert is_tool_simulated("Non Existent Tool") is False + + clear_execution_context() From 2f9e1863bf53a478941ee40d6e6d0eed3f5e728b Mon Sep 17 00:00:00 2001 From: Mayank Jha Date: Fri, 20 Feb 2026 16:10:29 -0800 Subject: [PATCH 2/2] fix: apply case-insensitive comparison in LLMMocker.response The LLMMocker had its own raw name comparison that bypassed _normalize_tool_name, causing simulation to fail when tool names differ in case (e.g. "API_Workflow" vs "API Workflow"). Co-Authored-By: Claude Opus 4.6 --- src/uipath/_cli/_evals/mocks/llm_mocker.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/uipath/_cli/_evals/mocks/llm_mocker.py b/src/uipath/_cli/_evals/mocks/llm_mocker.py index aadfb45e9..2888a19b5 100644 --- a/src/uipath/_cli/_evals/mocks/llm_mocker.py +++ b/src/uipath/_cli/_evals/mocks/llm_mocker.py @@ -90,7 +90,11 @@ async def response( assert isinstance(self.context.strategy, LLMMockingStrategy) function_name = params.get("name") or func.__name__ - if function_name in [x.name for x in self.context.strategy.tools_to_simulate]: + normalized_function_name = function_name.replace("_", " ").lower() + if normalized_function_name in [ + x.name.replace("_", " ").lower() + for x in self.context.strategy.tools_to_simulate + ]: from uipath.platform import UiPath from uipath.platform.chat._llm_gateway_service import _cleanup_schema