From 20b9888fb461f45ecc592428dbf432fc2fb57a9a Mon Sep 17 00:00:00 2001 From: GabrielVasilescu04 Date: Fri, 20 Feb 2026 14:42:08 +0200 Subject: [PATCH] feat: add attachments support for coded conversational agents --- pyproject.toml | 2 +- samples/chat-hitl-agent/graph.py | 2 +- .../file-attachments-chat-agent/.env.example | 3 + samples/file-attachments-chat-agent/README.md | 42 ++++ .../file-attachments-chat-agent/agent.mermaid | 9 + .../langgraph.json | 5 + samples/file-attachments-chat-agent/main.py | 32 +++ .../pyproject.toml | 10 + .../file-attachments-chat-agent/uipath.json | 14 ++ src/uipath_langchain/chat/__init__.py | 12 +- src/uipath_langchain/chat/tools/__init__.py | 21 ++ .../chat/tools/attachments.py | 211 ++++++++++++++++++ src/uipath_langchain/chat/{ => tools}/hitl.py | 0 uv.lock | 2 +- 14 files changed, 361 insertions(+), 4 deletions(-) create mode 100644 samples/file-attachments-chat-agent/.env.example create mode 100644 samples/file-attachments-chat-agent/README.md create mode 100644 samples/file-attachments-chat-agent/agent.mermaid create mode 100644 samples/file-attachments-chat-agent/langgraph.json create mode 100644 samples/file-attachments-chat-agent/main.py create mode 100644 samples/file-attachments-chat-agent/pyproject.toml create mode 100644 samples/file-attachments-chat-agent/uipath.json create mode 100644 src/uipath_langchain/chat/tools/__init__.py create mode 100644 src/uipath_langchain/chat/tools/attachments.py rename src/uipath_langchain/chat/{ => tools}/hitl.py (100%) diff --git a/pyproject.toml b/pyproject.toml index caec15c2e..3c06edf8a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "uipath-langchain" -version = "0.5.78" +version = "0.5.79" description = "Python SDK that enables developers to build and deploy LangGraph agents to the UiPath Cloud Platform" readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11" diff --git a/samples/chat-hitl-agent/graph.py b/samples/chat-hitl-agent/graph.py index 828fd7f45..0f93c361a 100644 --- a/samples/chat-hitl-agent/graph.py +++ b/samples/chat-hitl-agent/graph.py @@ -1,7 +1,7 @@ from langchain_anthropic import ChatAnthropic from langchain_tavily import TavilySearch from langchain.agents import create_agent -from uipath_langchain.chat import requires_approval +from uipath_langchain.chat.tools import requires_approval tavily_tool = TavilySearch(max_results=5) diff --git a/samples/file-attachments-chat-agent/.env.example b/samples/file-attachments-chat-agent/.env.example new file mode 100644 index 000000000..ae00c4096 --- /dev/null +++ b/samples/file-attachments-chat-agent/.env.example @@ -0,0 +1,3 @@ +UIPATH_ACCESS_TOKEN=YOUR TOKEN HERE +UIPATH_URL=https://alpha.uipath.com// +OPENAI_API_KEY=your_openai_api_key diff --git a/samples/file-attachments-chat-agent/README.md b/samples/file-attachments-chat-agent/README.md new file mode 100644 index 000000000..332f2449a --- /dev/null +++ b/samples/file-attachments-chat-agent/README.md @@ -0,0 +1,42 @@ +# File Attachments Chat Agent + +An AI assistant that reads and analyzes file attachments shared in the conversation. + +## Requirements + +- Python 3.11+ +- OpenAI API key + +## Installation + +```bash +uv venv -p 3.11 .venv +source .venv/bin/activate # On Windows: .venv\Scripts\activate +uv sync +``` + +Set your API key as an environment variable in .env + +```bash +OPENAI_API_KEY=your_openai_api_key +``` + +## Usage + +**1.** Upload the file to Orchestrator using the [attachments API](https://uipath.github.io/uipath-python/core/attachments/). + +**2.** Run the agent, passing the attachment ID and file metadata returned by the upload: + +```bash +uipath run agent '{ + "messages": [ + { + "type": "human", + "content": [ + { "type": "text", "text": "Summarize this document." }, + { "type": "text", "text": "[{\"id\": \"{orchestrator_attachment_id}\", \"full_name\": \"{file_name}\", \"mime_type\": \"{file_mime_type}\"}]" } + ] + } + ] +}' +``` diff --git a/samples/file-attachments-chat-agent/agent.mermaid b/samples/file-attachments-chat-agent/agent.mermaid new file mode 100644 index 000000000..495db26c9 --- /dev/null +++ b/samples/file-attachments-chat-agent/agent.mermaid @@ -0,0 +1,9 @@ +flowchart TB + __start__(__start__) + model(model) + tools(tools) + __end__(__end__) + __start__ --> model + model --> __end__ + model --> tools + tools --> model diff --git a/samples/file-attachments-chat-agent/langgraph.json b/samples/file-attachments-chat-agent/langgraph.json new file mode 100644 index 000000000..408405763 --- /dev/null +++ b/samples/file-attachments-chat-agent/langgraph.json @@ -0,0 +1,5 @@ +{ + "graphs": { + "agent": "./main.py:graph" + } +} diff --git a/samples/file-attachments-chat-agent/main.py b/samples/file-attachments-chat-agent/main.py new file mode 100644 index 000000000..5c88c7005 --- /dev/null +++ b/samples/file-attachments-chat-agent/main.py @@ -0,0 +1,32 @@ +from langchain.agents import create_agent +from langchain_openai import ChatOpenAI + +from uipath_langchain.chat.tools import AnalyzeAttachmentsTool + +system_prompt = """ +You are an AI assistant specialized in analyzing user-provided files using the available file analysis tool. +Always use the provided tool to read and analyze any uploaded or referenced file. Never guess or fabricate file contents. If a file is missing or inaccessible, ask the user to upload it again. + +When a file is received: + 1.Identify the file type. + 2.Provide a clear, concise summary. + 3.Extract key information relevant to the user’s request. + 4.Highlight important patterns, issues, or insights when applicable. + 5.If the user’s request is unclear, ask a focused clarification question before proceeding. + +For follow-up questions: + 1.Base all answers strictly on the file contents. + 2.Maintain context across the conversation. + 3.Perform deeper analysis, comparisons, transformations, or extractions as requested. + 4.Clearly distinguish between observed facts and inferred insights. If something cannot be determined from the file, state that explicitly. + +Keep responses structured, concise, and professional. Treat all file data as sensitive and do not retain or reuse it outside the current conversation. +""" + +llm = ChatOpenAI(model="gpt-4.1") + +graph = create_agent( + llm, + tools=[AnalyzeAttachmentsTool(llm=llm)], + system_prompt=system_prompt, +) diff --git a/samples/file-attachments-chat-agent/pyproject.toml b/samples/file-attachments-chat-agent/pyproject.toml new file mode 100644 index 000000000..b70c779ca --- /dev/null +++ b/samples/file-attachments-chat-agent/pyproject.toml @@ -0,0 +1,10 @@ +[project] +name = "file-attachments-chat-agent" +version = "0.0.1" +description = "file-attachments-chat-agent" +authors = [{ name = "John Doe", email = "john.doe@myemail.com" }] +dependencies = [ + "langchain-openai>=1.1.9", + "uipath-langchain", +] +requires-python = ">=3.11" diff --git a/samples/file-attachments-chat-agent/uipath.json b/samples/file-attachments-chat-agent/uipath.json new file mode 100644 index 000000000..7969b8f00 --- /dev/null +++ b/samples/file-attachments-chat-agent/uipath.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://cloud.uipath.com/draft/2024-12/uipath", + "runtimeOptions": { + "isConversational": true + }, + "packOptions": { + "fileExtensionsIncluded": [], + "filesIncluded": [], + "filesExcluded": [], + "directoriesExcluded": [], + "includeUvLock": true + }, + "functions": {} +} diff --git a/src/uipath_langchain/chat/__init__.py b/src/uipath_langchain/chat/__init__.py index 69e0e2b98..a714476ca 100644 --- a/src/uipath_langchain/chat/__init__.py +++ b/src/uipath_langchain/chat/__init__.py @@ -26,9 +26,17 @@ def __getattr__(name): return UiPathChatOpenAI if name == "requires_approval": - from .hitl import requires_approval + from .tools.hitl import requires_approval return requires_approval + if name == "resolve_attachments": + from .tools.attachments import resolve_attachments + + return resolve_attachments + if name == "AnalyzeAttachmentsTool": + from .tools.attachments import AnalyzeAttachmentsTool + + return AnalyzeAttachmentsTool if name in ("OpenAIModels", "BedrockModels", "GeminiModels"): from . import supported_models @@ -51,4 +59,6 @@ def __getattr__(name): "LLMProvider", "APIFlavor", "requires_approval", + "resolve_attachments", + "AnalyzeAttachmentsTool", ] diff --git a/src/uipath_langchain/chat/tools/__init__.py b/src/uipath_langchain/chat/tools/__init__.py new file mode 100644 index 000000000..d33eb9335 --- /dev/null +++ b/src/uipath_langchain/chat/tools/__init__.py @@ -0,0 +1,21 @@ +def __getattr__(name): + if name == "AnalyzeAttachmentsTool": + from .attachments import AnalyzeAttachmentsTool + + return AnalyzeAttachmentsTool + if name == "requires_approval": + from .hitl import requires_approval + + return requires_approval + if name == "resolve_attachments": + from .attachments import resolve_attachments + + return resolve_attachments + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +__all__ = [ + "AnalyzeAttachmentsTool", + "requires_approval", + "resolve_attachments", +] diff --git a/src/uipath_langchain/chat/tools/attachments.py b/src/uipath_langchain/chat/tools/attachments.py new file mode 100644 index 000000000..d68edb8f2 --- /dev/null +++ b/src/uipath_langchain/chat/tools/attachments.py @@ -0,0 +1,211 @@ +"""Attachment resolution for conversational agents.""" + +import asyncio +import json +import re +import uuid +from typing import Any, Type, cast + +from langchain_core.language_models import BaseChatModel +from langchain_core.messages import ( + AnyMessage, + ContentBlock, + HumanMessage, + SystemMessage, +) +from langchain_core.runnables.config import var_child_runnable_config +from langchain_core.tools import BaseTool +from pydantic import BaseModel, ConfigDict, Field, PrivateAttr +from uipath.platform import UiPath +from uipath.platform.attachments import Attachment + +from uipath_langchain.agent.multimodal import FileInfo, build_file_content_block +from uipath_langchain.chat.helpers import ( + append_content_blocks_to_message, + extract_text_content, +) + +_ATTACHMENTS_PATTERN = re.compile( + r"(.*?)", re.DOTALL +) + +_READ_ATTACHMENTS_SYSTEM_MESSAGE = ( + "Process the provided files to complete the given task. " + "Analyze the files contents thoroughly to deliver an accurate response " + "based on the extracted information." +) + + +async def resolve_attachments(messages: list[AnyMessage]) -> list[AnyMessage]: + """Resolve file attachments embedded in conversational messages. + + UiPath's runtime injects attachment metadata as a ```` + text block into HumanMessages. This function resolves those references to + actual file content blocks so the LLM can read the files. + + Messages that contain no ```` tag are returned unchanged, + so it is safe to call this on the full message history on every turn. + + To persist resolved content to the checkpointer (avoiding re-downloads on + subsequent turns), include the resolved messages in the node's return value + alongside the LLM response. LangGraph's ``add_messages`` reducer replaces + messages with matching IDs rather than appending them:: + + async def call_model(state: MessagesState): + resolved = await resolve_attachments(state["messages"]) + response = await llm.ainvoke([SystemMessage(...)] + resolved) + return {"messages": resolved + [response]} + + Args: + messages: List of LangChain messages from the agent state. + + Returns: + New list where HumanMessages with attachments have their + ```` text replaced with DataContentBlock items + containing the actual file content. + """ + return list(await asyncio.gather(*[_resolve_message(m) for m in messages])) + + +async def _resolve_message(message: AnyMessage) -> AnyMessage: + if not isinstance(message, HumanMessage): + return message + + content = message.content + if not isinstance(content, list): + return message + + clean_blocks: list[Any] = [] + file_infos: list[FileInfo] = [] + + for block in content: + if not isinstance(block, dict) or block.get("type") != "text": + clean_blocks.append(block) + continue + + text = block.get("text", "") + match = _ATTACHMENTS_PATTERN.search(text) + if not match: + clean_blocks.append(block) + continue + + # Parse attachment metadata + attachments: list[dict[str, Any]] = json.loads(match.group(1)) + file_infos.extend(await _resolve_file_infos(attachments)) + + # Preserve any text outside the tag + remaining = _ATTACHMENTS_PATTERN.sub("", text).strip() + if remaining: + clean_blocks.append({"type": "text", "text": remaining}) + + if not file_infos: + return message + + file_blocks = list( + await asyncio.gather(*[build_file_content_block(fi) for fi in file_infos]) + ) + + return HumanMessage( + id=message.id, + content=clean_blocks + file_blocks, + additional_kwargs=message.additional_kwargs, + ) + + +async def _resolve_file_infos( + attachments: list[dict[str, Any]], +) -> list[FileInfo]: + client = UiPath() + file_infos: list[FileInfo] = [] + + for att in attachments: + att_id = att.get("id") + if not att_id: + continue + + blob_info = await client.attachments.get_blob_file_access_uri_async( + key=uuid.UUID(att_id) + ) + file_infos.append( + FileInfo( + url=blob_info.uri, + name=blob_info.name, + mime_type=att.get("mime_type", ""), + ) + ) + + return file_infos + + +class AnalyzeAttachmentsInput(BaseModel): + """Input schema for the AnalyzeAttachmentsTool.""" + + model_config = ConfigDict(arbitrary_types_allowed=True) + + query: str = Field(description="What you want to know about or do with the files.") + attachments: list[Attachment] = Field( + description="The attachment objects from inside the tag." + ) + + +class AnalyzeAttachmentsTool(BaseTool): + """Tool that reads and interprets file attachments using the provided LLM. + + The tool downloads each attachment, passes the file content to a non-streaming + copy of the provided LLM for interpretation, and returns the result as text. + This keeps multimodal content out of the agent's message state — the original + ```` metadata in HumanMessages is never modified. + + Example:: + + from langchain_openai import ChatOpenAI + from uipath_langchain.chat import AnalyzeAttachmentsTool + + llm = ChatOpenAI(model="gpt-4.1") + tool = AnalyzeAttachmentsTool(llm=llm) + """ + + name: str = "analyze_attachments" + description: str = ( + "Read and interpret the content of file attachments provided by the user. " + "Call this when you see a tag in a user message, passing " + "the attachment objects from inside the tag and a query describing what you " + "want to know about or do with the files." + ) + args_schema: Type[BaseModel] = AnalyzeAttachmentsInput + + llm: BaseChatModel + + _non_streaming_llm: BaseChatModel = PrivateAttr() + + def model_post_init(self, __context: Any) -> None: + self._non_streaming_llm = self.llm.model_copy( + update={"disable_streaming": True} + ) + + def _run(self, query: str, attachments: list[Attachment]) -> dict[str, str]: + raise NotImplementedError("Use async version via arun()") + + async def _arun(self, query: str, attachments: list[Attachment]) -> dict[str, str]: + file_infos = await _resolve_file_infos( + [a.model_dump(mode="json") for a in attachments] + ) + if not file_infos: + return {"analysisResult": "No attachments provided to analyze."} + + file_blocks = list( + await asyncio.gather(*[build_file_content_block(fi) for fi in file_infos]) + ) + + human_message_with_files = append_content_blocks_to_message( + HumanMessage(content=query), cast(list[ContentBlock], file_blocks) + ) + + messages = [ + SystemMessage(content=_READ_ATTACHMENTS_SYSTEM_MESSAGE), + human_message_with_files, + ] + + config = var_child_runnable_config.get(None) + result = await self._non_streaming_llm.ainvoke(messages, config=config) + return {"analysisResult": extract_text_content(result)} diff --git a/src/uipath_langchain/chat/hitl.py b/src/uipath_langchain/chat/tools/hitl.py similarity index 100% rename from src/uipath_langchain/chat/hitl.py rename to src/uipath_langchain/chat/tools/hitl.py diff --git a/uv.lock b/uv.lock index 7f39a6e84..f9bd18dbc 100644 --- a/uv.lock +++ b/uv.lock @@ -3323,7 +3323,7 @@ wheels = [ [[package]] name = "uipath-langchain" -version = "0.5.78" +version = "0.5.79" source = { editable = "." } dependencies = [ { name = "httpx" },