From 073b6996395e352eaad56374be707d863b3e9637 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Gonz=C3=A1lez=20Alonso?= Date: Tue, 10 Feb 2026 15:39:51 +0100 Subject: [PATCH 1/5] feat: add methods to create and execute langgraph AI agent --- .vscode/launch.json | 2 +- .vscode/settings.json | 8 -- CHANGELOG.rst | 3 +- VERSION | 2 +- docs/ai_utils.rst | 34 ++++++ docs/react_agent.png | Bin 0 -> 10679 bytes pyproject.toml | 4 +- requirements_ai.txt | 5 +- toolium/test/conftest.py | 64 +++++++++++ toolium/test/utils/ai_utils/test_ai_agent.py | 80 +++++++++++++ toolium/utils/ai_utils/ai_agent.py | 115 +++++++++++++++++++ 11 files changed, 303 insertions(+), 14 deletions(-) create mode 100644 docs/react_agent.png create mode 100644 toolium/test/conftest.py create mode 100644 toolium/test/utils/ai_utils/test_ai_agent.py create mode 100644 toolium/utils/ai_utils/ai_agent.py diff --git a/.vscode/launch.json b/.vscode/launch.json index eace506d..cc400609 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -8,7 +8,7 @@ "program": "${file}", "console": "integratedTerminal", "justMyCode": false, - "envFile": "${workspaceFolder}/.vscode/.env" + "envFile": "${workspaceFolder}/.env" } ] } diff --git a/.vscode/settings.json b/.vscode/settings.json index d7bb32a5..09c80098 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -23,14 +23,6 @@ "python.testing.pytestArgs": [ "." ], - "python.envFile": "${workspaceFolder}/.vscode/.env", - "python-envs.pythonProjects": [ - { - "path": "", - "envManager": "ms-python.python:venv", - "packageManager": "ms-python.python:pip" - } - ], "ruff.organizeImports": true, "cucumberautocomplete.steps": [ "steps/**/*.py", diff --git a/CHANGELOG.rst b/CHANGELOG.rst index da74c3eb..bc206a5d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,7 +1,7 @@ Toolium Changelog ================= -v3.7.1 +v3.8.0 ------ *Release date: In development* @@ -10,6 +10,7 @@ v3.7.1 - Configure ruff for linting and formatting files, replacing flake8 and black - Add text analysis tool to get an overall match of a text against a list of expected characteristics using AI libraries that come with the `ai` extra dependency +- Add langgraph methods to create a ReAct AI agent to test the behavior of other AI agents or LLMs v3.7.0 ------ diff --git a/VERSION b/VERSION index bafa3228..a2c8d936 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.7.1.dev0 +3.8.0.dev0 diff --git a/docs/ai_utils.rst b/docs/ai_utils.rst index 09c5debb..852828b8 100644 --- a/docs/ai_utils.rst +++ b/docs/ai_utils.rst @@ -226,3 +226,37 @@ custom behavior after accuracy scenario execution, like calling Allure `after_sc # Monkey-patch the hook accuracy.after_accuracy_scenario = custom_after_accuracy_scenario + + +AI agents for testing +--------------------- + +Toolium provides utilities to create and execute AI agents in your tests using langgraph library, allowing you to +simulate complex user interactions or validate AI-generated responses. + +You can create an AI agent using the `create_react_agent` function from the `toolium.utils.ai_utils.ai_agent` module. +This function allows you to create a ReAct agent, which is a type of AI agent that can reason and act based on the +conversation history and tool interactions. You must specify the system message with AI testing agent instructions +and the tool method, that the agent can use to send requests to the system under test and receive responses. + +.. image:: react_agent.png + :alt: ReAct Agent Flow Diagram + +Once you have created an AI agent, you can execute it using the `execute_agent` function from the same module. This +function will run the agent and log all conversation messages and tool calls, providing insights into the agent's +behavior and the interactions it had during execution. +You can also provide previous messages to the agent to give it context for its reasoning and actions. + +.. code-block:: python + + from toolium.utils.ai_utils.ai_agent import create_react_agent, execute_agent + + # Create a ReAct agent with a system message and a tool method + system_message = "You are an assistant that helps users find TV content based on their preferences." + tool_method = tv_recommendations # This should be a function that the agent can call as a tool + model_name = 'gpt-4o-mini' # Specify the model to use for the agent + + agent = create_react_agent(system_message, tool_method=tool_method, model_name=model_name) + + # Execute the agent and log all interactions + final_state = execute_agent(agent) diff --git a/docs/react_agent.png b/docs/react_agent.png new file mode 100644 index 0000000000000000000000000000000000000000..41354069c90d429eae519a3a8bd88acbff4c0956 GIT binary patch literal 10679 zcmbt)Wmp`+w(a0ioTeNIRF{T#Yl8vY&XI^fG{>^^vU zdDTL%JP`{GG-=Y;lt*8WL!)D3>U-kzRMj=)HAd*?TfDG7XctBJFj~vD=P_ac;K{NrP zI6`o5HygS2YKP3)*47}<&~x@gexLoj`PMBOjY5GK*rFgV!&5#^D0*S&wuwf8cs7iL ztM28hwCnZiT)ehuKiriX7U{ta*e@w5#S#2@Y3Aee_4Pe9l|D)`S=yVxP0W#^&Tt-5 zF2S&S)ZlgxG&A@Mpdv9XO+_<>Tvu1uHOaW7qJn*9#)*WUejM&fjzJtphRS&=9It&n?yxr#c@Kw^7X`P0{~nEvjjoIDl9S`VeQbX2y5OJ-@=z^=G+p9R zaB*_h+MhXS6rD3{OP{ktq0r*FgOW+7li6IW;<7SI$Fck_tmDqKwzgo(6lHVKFqE41 zYt9cEg~>!KVf4AY_99`OOe(3?$o<*B#**`zhhMA)5t%i)#hCNPne+As4d+03gsNt3 zYnkQbq8P^08|>dIq4Yvx-VMAPzxnv-Q+H>PSu_ESi1p7ed)WB2@wt}4C^CqpK|2$LBnD_JMwcn3wytH!ZkOsNssl3)^HAwvT`6MIP?v*W&^ssRyVvK-?eOZzD4I{{vj3H$0w2c@Jx-~G*M zw!h;kG+)#MH3AK7{otU75Vn1A5T#qHy#8x2Yrffo$?)?fF=sZebION+|NJ7}>G|TZEc2r-ppi2Q z?%ujm6^|QOM*8rUzwe}%s+yWy?|^$W5s^}jW&7IJ@+s?JBnF2YrnKfmiD zw70Ksqx$Qy&9HD%YNUs&elK?0ec#)BtJKqdt{UsPYO4hSd3pKt?Ch8~Z>ZrhyQ5}& z0l)2bk%8>loT8%fi&FCM=MHp@jIgk<;t%e$PHz*kpzM#dA85|>80l2Sk4dU%s^a}a0I+~=C2olZi;^$GkFCvmyGP&NP_k;} zN3$IB+M5Zdp$;TZSnz+wTYg+>>rJ_m9DII7flWx*)zbi$*MmMQvh1CZ6$$;`{AQ@1R`CM<$J&5fuw6NtmS}($3^9?~F zuQ*wNAy}L1C*pmCa~XVnGf67$Rh@2Gqw=3njl6RppF|GP)8U6=4(4Fw@Tf{%>yAwWt)_^Oz z68{fvucN0O?aBcr86{()nK25-|2Ia#hDiYK&WR^@2V%@0Q0*M}=jH$59o;z61z3uI z|BVlX$bV3hg#XsSKX^)hu}}%l-ybDWPdnEY0|KbbP(iP9 za_U*2-`T@e)6_LIqV+fMGukaGo4j_$8b7MBjIpz`TX$mALCIvxRAORcY@YvdfG<^R zPQE1}C-2Kicw7o%lyW;qh#9wfQ<=Hh?@igimI@*?V>L`QGAn;tuOuJXy?M61QC<{k zH~hDzV-;S&LrNoyqNt%JgaUpv!9fR?{c{3pZ)KdHM8D}NhmB5A?I-Kl zyA%&rEAPIOMb|p+A*}n0`p=f%q~xQ;V24R#*LU#(NZgozYJM{@&lyGS`GG&&)J|(U zdBC({ip%}}nb`HpR+J4MRxRwU(oMLiez083A9^o72b1{O7;T!-fNr!jzr(5)8;gMH zSl3#-K5jg@>ohU;j$-H{Fu~&$Ogeu-CtSls1LqrfDknV2d%kuVK2Dnx5Bm%|pYEOP z0Y^F9FdgKY!|nf)2mMz%ghi@Ea94wQzV*|e9gRn3j+cU`34dgD<;DIP(x-CfOb-#| z@5f@*9s5lLREUC{hFuowM*$~e^?(&=n~Cg`94G_B@M(+C2^TE?uT!H;CENa!N2sIt zzKgeO5-~2rRD%GKx!k_CXT=%kydh0Ci>bMIX4=prp2d9uivqA1TaA2%a$LtK2?p_= zZfOz0cLeO>ol!d1qH`&8JAT;1g`oP>qE&4Nm$MKu{)Ur;;=-WVjx!uGlC@CQe9GK%30;)3Lc)iOCBB{avEj-msKhv!09+3Cpa?iCYkP(@|6%Fh+o z{CiHET&Nd>ctLLa7!{lrx6~|eM>i7FZ!I#vfLAR6=YAg3OSe~{udh(~Ny*-XKoYYh za(rj?r0)GgC{=*tV`5wk(MAXkhm52AE_?LIDu$1Nc>WFRW0pPj8N(Tc0wa`1d zZsYChtA$bNH2pe-RVW~H>6MAO1AIz> zMw#luCw&nt55=F%lUnx!X-$1`2W5u&*R*6*Z_V9rOB+-dgK$8_ZST6W!*GkI_a$f- z3sb>nx_UTMp`+dF295x{0>vj6pPmjRz8|LJM!_JC=(qMu4hteXvUm#ap~vaIY*OiQ z5`4%0krI?1ri`LOgq0iwp;aiy+E__@M|`CxO3TcsZ&{^$WiKS}hz0RCWN-@$KtHkw z!n$P&YLMBEwGF~(few5Dfxe}`K*0|4=0hMb=%j@~Yj{`9LEmA`X9bhZFPHtB&$Egd zEKxEVh3r|tz4lN!q$vQP)SyB^PaF3LZdo?A6ufy7(xrBgJTf1Ng(Rr#bPX`*mBbh9G4BPxtE)_;}sIU~P=;X<@CbfhH zeX)5@4?-cmbAPy&fSh`DUBM_$hr91C?_|PCKk4HIu|bYZeI(?;RLPn6EQIZTxH2ON za;6dzAuk=cCtCWIxoI%cQz~`mHm160@mtGfz$=~jFWc#6{vhS+KC=-b91k899V9 zo1Sn9K-m4FX>hf(Ywe-mfzvmM?v#HLSd6jXDt+25f_U{Lvn)OXmmCxk4Yvz9KwRBO z-*KnH9N$Ztre{^iUO7#S==J;vfWvi<9e&|7MPZM<>!QPCLc9Zmi?(#8u!EeyHCT*b9N(g@NU4 zLRBt4US4r0@Qf&EK5*2|DpWyHbW3FHpv`?~ORO7?Trf=I=`9XX7HoL%=U z^pgky&~*#JYEdC$7heu(dVp9cJZ12FPAE;0R*`K(+=4_9S{ZndLz_DBVM;d#7xOqu z&0&*TqF><_w;fNtX90xJNNvTXyJZXP4$Bmgh<*(k4OcCfxXcK2O8j=nj%woD^v5BaVSOHPV*V>0c8k};X83yT}3G*cX8%cePhf6}qg!>b{Y z3gVCpTq4vr%26;g!G#%!IdcwE%NW)v@LxxKrxb;Xn{~Wk8RRs-uL2s1O8}J&`ZM zk==;ZV1G2UAiRf3Ci|;ZugRg5TeD|xq#P%$KTeiCwz@hNhGeh%irro+>9N;@N>~{hhx0b*)9V4`3i0KUl%b=J+@;#}DI%DupUI1z{jl34qch zR;$OI6%FfVmdW?NRba@3haJ%J#?!u^FsxjtS8E2Ge%#ovEHsNiDKZn5#S8GJ(V-{K z?#=*drq|)LAS5&h*46_Qkf1nTUYc2iF2}|_B0sz9uS2rpwhgm+$i~wAORit{> zuD+zZo#NaW$@A9y+ZSne?dMD)@OW6pTVO-2)Oi-Edc#Tsns@+PCHsq2^w*rW8ffWR z>9ga_3$O+EH8UL>^E<1*c#bgC0WB0GR`J|@@0756Q3p`-#Y`+4xtNdcCtg|D4Ckl+ z@rqTcz&8o{y+Gaq>^=%JU(}-dWQ*(nnzJ6*yyM@^=pddq8+#^e?MOsnBvX+{P^Otc+VLToZjP%FDV*2qjRSUJb3rv6VA}Fth7@9Fn98Ls1&Zv^>BL(3j zc^%K&N?Ocu41`A`TO&~ihIS9|*)?@HkB!Dk*8FjT`vSgsHIH!fmhTd_-$v@bDMF7* z;MF+9e^9fWFgREc*h4oZ8c+v9QS*rsD<|G3zN0AO8KXT2opnKyGYi5hO&aZG+(r1L zYZ56i9n6<<0FsBwKs>qn=dD~P51x^Qe$+mcN5I!IqIVpnbcGgG^EcYXWUun*t|OelpP zF!zSwqBa#G#jVD8b#JkK3E>yP-j7GaxfM;`U^>n@u5G}F-GBRp8%5J#F73Evt)qcL zVv|+0Ou5ar@vR0_`qBc}pYa!&yIi8V+{{RnfdOYYoTaZW zj2KXp=!J{g9y7}Nju^l-+`>@@G-hAyXEgKUErglJ?6@0=Uic~&Nk}c&2v=DRNHu$} z`%w~XdtfgCkq{sGg=yE}??V6ibIYWqJ0-qd{1e)B!V8-p9`{<>YSPpM8Kq=$rC#>Csh1}1#xI3`y@sx0Z`d_l?-rG7R;l&c#i2p2V%ww zY=!sMy=xXQzvRRtEQE>%CagOW-9Rv-m*b@6{AyuY{4+EqG zn<8ktj%D(`omRa(V|w+gy!H^hGqZ?dfZJ-T>^LdSKq_7*h%$^gzK(b7B%wU}CNUuSl_t;8ui+5v z^XonOuJgzy!qrO!#ST?H00SS1i9TLF%KO|CNgZD|og@D|Cn@^wK-2fR0ubbvtIL5M z7YzpFqAL();+GU%lm75!M4>Ys2U1n9$cf&cNh}ARI)^k8eq6QNoaz9gHn$Qm-a_w9 zXE+ZSyz6Y^vV13`i`{gL-uHB|X(@!Xc;RKOo`&_Ti*zONWPv|$49oT9SrymJF7@km+7G73^ zMZ{UFRo?PB1WOyscI4aCM{OFX##Kc1B0_wmN??+Kei}@iS(?exc&1M+)yJ2vs0JM|}U*^!1DBwx^fWS@k%F@XxLbdl`U24Zt*nfprHO&;5gpa=VXkut#t*F2M=wPO+t$}MP_P_D%9L`~r=EO&eX^%|sdx_Ud~EZYJAB4j zh%Aw@Lf_e`(G(7AF}*vdmU(e3GUT$8W!P9vlJq@&u~ z=rR%bw$oWJ(5<5<{SOKFP&MhB{Ko<$EqhpLh}I&Az-e>M82v7%OMX3B;}GyAvBV(5 zo{vlU<3CQI!7ZwJIE>~Tu|n`(Js~}v_!4;Ow3@kwS6j3hlR-J!D(7q zC;Z~r3qER^qKzNx)8TWMOTVeTV&~3b}|20QU>g4*_f7b9`8>mZlJT}rJVho--b4TJp7n_P9)^7^UY?DmV zCY_vao4=opZAM^O_5^{6B*C3mn=UOgXxh>$uVOs&)9xPK#cFodN|4td5?X{&Pbp1K zz9oJniT~&$S^0SHCZGUTNdz0u1;H$8R#N19O2zHZH4IyQWNRX#gBfnGECZ_*-sRbsNHeEQC?)NCY@x(NA!!8!0}6<<1U!2{iXrt6Zhm)K%!iQ8M0_CFhVfSAxX-g)b6 zwjQ-DlUOC~S{N(x(4Q<4F;S~?dplh7xBl6aj&JZP<;gAoOZ;TJ;pH9EP(c>eq&z}; z$f^=5@4yY&qW+fY%G;zqNvpMDvE(*0zmqC(Ea3OF*};z!I~(E!m2wxNHK1@CxzR;eIx&2udy$5EW_tOrmL?;WMiT$$~DP* zsFB#2RB`ks`i*dVHGwPm<@a?jE;q4o@+YBqz?NH7F2!by#g30zwO!1lW%i0BDMBYEId8&0bhk5AR-f6pns83hd<2x z`0=i+j7+>B6_)q@za-VLGZpUmJ*q!QfG5XPuZYjRKJo<7%|B4EYQE{sfsLJR_LKPZ z!Ua_k2{h+5-p{{R>RU-TU!l7 zo)MD?V*kb$0}Nw$vMGvNucc5^sj!~_OJ_r%fG7zt&DEsi6&5YK?yJU`m_jMMeX5M2 zD)b*i?#HzOwXU!`L5i!Gd&GJ6<)9NE1KS;Y>U|qJ|K#{a_qt}qwC|Av_IAZEN|_Fs z$xMk~fmWPfE_WB;tDppZ+ywo^Im-}?XR9F;fc|U>;|X2-NfX=@CMLxgTLRiH|%U};l#X)=gD$&n3&s~Iev)q zitC7MpT{BOtyaC;38H=Q;n;VnL;9dH5{_dd}P?PEol>o^O zB^;{8Ur|C;ME=kTqli(Yt^F!|43i(#13LSQX_Zm|GXK@9=#ZPxp8lfXL;@ONXisd4 z@z--E#7aA!nwr|^W~4%7-9=M~@xy8V?coN+S%L7S9SQlM&4eXRrQ&mwn(DMMFqMq)1rdXn{O11%y4{6+`OvZeFhmgW7xG{ zcUvORRUAcLaS7N63a&gKaf)W#K}8@6)|p{JOlYOLM636u{2oUtuTM9|vA>?Fz1k|5 zGw{r#b0%q}mCdka@(_kZ@Vu=$IUzYg9;R#!%gFkk+XEumI-rPm4Unu4s2+=|21tUB zS}rWh%F3*#KWMUZbIZHBa)-{hMA12)kFhc{c)AktTG^B`ABEUV9OzW>D#J8cB(G!T ze*H?F3G0Ya&mi!5!JGMTMUE-~xy4kHWA{`nDqhrmlbe_vEmfhHyXHA0XFi&!aQuxQ zR((=4ZMd=K_t3Q}(}Ce*BE9ciS*&jt^^z@mVx?}@_}SzZsP*6|wp1?XozWA&;Hz28 zidkxnd_FJgFL@+C^3wVW%E<7usu3!BBM(2o1~g4k&ULbmq$v7)oWTodyM$zoyV$D)Hy>0 ziLYb7onWO0-%8w`c4DZCo`?k!(Z;P+T*0cD8Ym=ee=R?eHGAK6&od4k-|=!sPH{vp zNz)6YnVHcRay5Nn+^C2r@xMzE(F_z*_S6$_7%v1gdH7=gW?xFrU>0ngl1{>xTlnPy zE?apl43?sd*1iJD)A;FIccN8MRi&h)#A0q?PmGN|iZrqc!1(HW_tTjh1Y}GxaB)>G z_tUlq;IV2Kj^qT50Q&sNThr;65q2}|A_b4M4zHfL|CwMS)y#KHwej>SqU#@h+1CzDN zqx3#HxACaENP=sYTXonGp7TrifVEZdLm`Q|Aaf8ywmQQjpWDa-UTx_sbceBEY;7r| z;9|Y;Hn}W(=od5W}i#8MqR*BLM8I+If*msjrnrgzP^H)Vbo=Dec~Dh@VjJx zNFctze8DrWl_eas)-c{F>+X&XE<_S%KB}=WQ)t@DCnTSeFfwZl;~FSJN#k)Y}uITpvamD zSO8=k*o^GzPE^dQYN<7T>N%X-+@p`}D&1yq8;{c=pAuL3P{#d=3Amw@bS9L%tFiE2 zG-(Q^yjTA98{c&iP7vgK9IN+UQB@WD_WH6*iSdOGDt$}GL4gc92r~-hF*U*O-)HIh z`S?b8Tpz|?M+BU0n=C;+mLUgAU}o?Fc0xa>IS@1arDFgEt{HArHF#*_*~BY5Cf9C8 z^wig5z+w7dAfy50s$*(&mv_TCSw4)8u&9`n;LqnlSum9UTG7aO*~+hq6&I zx4RuaBF_4oNKo4vLJyq0fgH2{N5;Z`Ok?=h!jT8Gk?9~`v;(f6K(ZxBO-W0!THZSH Fe*v~d Date: Tue, 17 Feb 2026 15:37:07 +0100 Subject: [PATCH 2/5] update dependencies in CI --- .github/workflows/ci.yml | 1 + .vscode/settings.json | 7 +++++++ requirements_dev.txt | 3 --- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 66b4a850..5bcc2a0e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,6 +20,7 @@ jobs: run: | python -m pip install --upgrade pip pip install -r requirements.txt + pip install -r requirements_ai.txt pip install -r requirements_dev.txt python -m spacy download en_core_web_md - name: Lint and format check with ruff diff --git a/.vscode/settings.json b/.vscode/settings.json index 09c80098..850a0b5e 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -23,6 +23,13 @@ "python.testing.pytestArgs": [ "." ], + "python-envs.pythonProjects": [ + { + "path": "", + "envManager": "ms-python.python:venv", + "packageManager": "ms-python.python:pip" + } + ], "ruff.organizeImports": true, "cucumberautocomplete.steps": [ "steps/**/*.py", diff --git a/requirements_dev.txt b/requirements_dev.txt index ee710060..bb00ed6e 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -9,7 +9,4 @@ wheel~=0.40 twine~=6.2 behave~=1.3 # behave tests importlib_metadata~=8.7 -spacy~=3.8 -sentence-transformers~=5.1 -openai~=2.7 ruff~=0.15 From 37546a01ed21670650b620cc407f690e6bbc6a46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Gonz=C3=A1lez=20Alonso?= Date: Thu, 19 Feb 2026 18:27:23 +0100 Subject: [PATCH 3/5] use logging.conf file --- toolium/test/conftest.py | 43 ++++++---------------- toolium/utils/ai_utils/ai_agent.py | 1 - toolium/utils/ai_utils/openai.py | 1 - toolium/utils/ai_utils/spacy.py | 1 - toolium/utils/ai_utils/text_analysis.py | 1 - toolium/utils/ai_utils/text_readability.py | 1 - toolium/utils/ai_utils/text_similarity.py | 1 - toolium/utils/poeditor.py | 1 - 8 files changed, 12 insertions(+), 38 deletions(-) diff --git a/toolium/test/conftest.py b/toolium/test/conftest.py index ac92018b..50e7f1c8 100644 --- a/toolium/test/conftest.py +++ b/toolium/test/conftest.py @@ -16,6 +16,7 @@ """ import logging +import logging.config import os import pytest @@ -23,42 +24,22 @@ def pytest_configure(config): # noqa: ARG001 """Configure logging for all tests in this directory and subdirectories.""" - # Configure logging to show DEBUG messages and save to file - log_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + # Configure the log filename (use forward slashes for cross-platform compatibility) + log_filename = 'toolium/test/output/toolium_tests.log' - # Console handler - console_handler = logging.StreamHandler() - console_handler.setLevel(logging.DEBUG) - console_handler.setFormatter(log_formatter) - - # File handler - log_dir = os.path.join('toolium', 'test', 'output') + # Ensure log directory exists before loading logging config + log_dir = os.path.dirname(log_filename) os.makedirs(log_dir, exist_ok=True) - log_file_path = os.path.join(log_dir, 'toolium_tests.log') - file_handler = logging.FileHandler(log_file_path, mode='w', encoding='utf-8') - file_handler.setLevel(logging.DEBUG) - file_handler.setFormatter(log_formatter) - - # Configure root logger - root_logger = logging.getLogger() - root_logger.setLevel(logging.DEBUG) - - # Clear any existing handlers to avoid duplicates - root_logger.handlers.clear() - - # Add our handlers - root_logger.addHandler(console_handler) - root_logger.addHandler(file_handler) - # Ensure specific toolium loggers use DEBUG level - logging.getLogger('toolium').setLevel(logging.DEBUG) - logging.getLogger('toolium.utils.ai_utils.ai_agent').setLevel(logging.DEBUG) + # Load logging configuration from .conf file with custom logfilename + config_file = os.path.join('toolium', 'test', 'conf', 'logging.conf') + logging.config.fileConfig(config_file, defaults={'logfilename': log_filename}, disable_existing_loggers=False) @pytest.fixture(scope='session', autouse=True) def setup_logging(): - """Session-level fixture to ensure logging is properly configured.""" - # This fixture runs automatically for all tests - # Additional logging setup can be done here if needed + """ + Session-level fixture to ensure logging is properly configured. + This fixture is automatically used for all tests in this directory and subdirectories. + """ yield # noqa: PT022 - # Cleanup can be done here if needed diff --git a/toolium/utils/ai_utils/ai_agent.py b/toolium/utils/ai_utils/ai_agent.py index 9e022d59..562ec46c 100644 --- a/toolium/utils/ai_utils/ai_agent.py +++ b/toolium/utils/ai_utils/ai_agent.py @@ -29,7 +29,6 @@ from toolium.driver_wrappers_pool import DriverWrappersPool -# Configure logger logger = logging.getLogger(__name__) diff --git a/toolium/utils/ai_utils/openai.py b/toolium/utils/ai_utils/openai.py index 0a8dde1d..ea33b68d 100644 --- a/toolium/utils/ai_utils/openai.py +++ b/toolium/utils/ai_utils/openai.py @@ -26,7 +26,6 @@ from toolium.driver_wrappers_pool import DriverWrappersPool -# Configure logger logger = logging.getLogger(__name__) diff --git a/toolium/utils/ai_utils/spacy.py b/toolium/utils/ai_utils/spacy.py index aefcba9c..044b1b2a 100644 --- a/toolium/utils/ai_utils/spacy.py +++ b/toolium/utils/ai_utils/spacy.py @@ -25,7 +25,6 @@ spacy = None -# Configure logger logger = logging.getLogger(__name__) diff --git a/toolium/utils/ai_utils/text_analysis.py b/toolium/utils/ai_utils/text_analysis.py index 8bafc36e..4693bed7 100644 --- a/toolium/utils/ai_utils/text_analysis.py +++ b/toolium/utils/ai_utils/text_analysis.py @@ -20,7 +20,6 @@ from toolium.utils.ai_utils.openai import openai_request -# Configure logger logger = logging.getLogger(__name__) diff --git a/toolium/utils/ai_utils/text_readability.py b/toolium/utils/ai_utils/text_readability.py index af9669a1..b9501ddc 100644 --- a/toolium/utils/ai_utils/text_readability.py +++ b/toolium/utils/ai_utils/text_readability.py @@ -20,7 +20,6 @@ from toolium.driver_wrappers_pool import DriverWrappersPool from toolium.utils.ai_utils.spacy import get_spacy_model -# Configure logger logger = logging.getLogger(__name__) diff --git a/toolium/utils/ai_utils/text_similarity.py b/toolium/utils/ai_utils/text_similarity.py index 85f3e2f3..0390beb0 100644 --- a/toolium/utils/ai_utils/text_similarity.py +++ b/toolium/utils/ai_utils/text_similarity.py @@ -27,7 +27,6 @@ from toolium.utils.ai_utils.openai import openai_request from toolium.utils.ai_utils.spacy import get_spacy_model, preprocess_with_ud_negation -# Configure logger logger = logging.getLogger(__name__) diff --git a/toolium/utils/poeditor.py b/toolium/utils/poeditor.py index 3c449547..398e2c22 100644 --- a/toolium/utils/poeditor.py +++ b/toolium/utils/poeditor.py @@ -70,7 +70,6 @@ ENDPOINT_POEDITOR_EXPORT_PROJECT = 'v2/projects/export' ENDPOINT_POEDITOR_DOWNLOAD_FILE = 'v2/download/file' -# Configure logger logger = logging.getLogger(__name__) From 6a5635384028333e4490cd1e3c717a53783c2f5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Gonz=C3=A1lez=20Alonso?= Date: Fri, 20 Feb 2026 10:54:29 +0100 Subject: [PATCH 4/5] allow azure and openai providers --- docs/ai_utils.rst | 9 ++++- toolium/test/conf/properties.cfg | 1 + toolium/test/utils/ai_utils/test_ai_agent.py | 4 +- toolium/utils/ai_utils/ai_agent.py | 40 +++++++++++++------- 4 files changed, 39 insertions(+), 15 deletions(-) diff --git a/docs/ai_utils.rst b/docs/ai_utils.rst index 852828b8..4350029f 100644 --- a/docs/ai_utils.rst +++ b/docs/ai_utils.rst @@ -254,9 +254,16 @@ You can also provide previous messages to the agent to give it context for its r # Create a ReAct agent with a system message and a tool method system_message = "You are an assistant that helps users find TV content based on their preferences." tool_method = tv_recommendations # This should be a function that the agent can call as a tool + provider = 'azure' # Specify the AI provider to use, e.g., 'azure' or 'openai' model_name = 'gpt-4o-mini' # Specify the model to use for the agent - agent = create_react_agent(system_message, tool_method=tool_method, model_name=model_name) + agent = create_react_agent(system_message, tool_method=tool_method, provider=provider, model_name=model_name) # Execute the agent and log all interactions final_state = execute_agent(agent) + +Default provider and model can be set in the properties.cfg file in *[AI]* section:: + + [AI] + provider: azure # AI provider to use, openai by default + openai_model: gpt-3.5-turbo # OpenAI model to use, gpt-4o-mini by default diff --git a/toolium/test/conf/properties.cfg b/toolium/test/conf/properties.cfg index 4ad50f22..f2e856bb 100644 --- a/toolium/test/conf/properties.cfg +++ b/toolium/test/conf/properties.cfg @@ -65,3 +65,4 @@ text_similarity_method: spacy spacy_model: en_core_web_sm sentence_transformers_model: all-mpnet-base-v2 openai_model: gpt-4o-mini +provider: azure diff --git a/toolium/test/utils/ai_utils/test_ai_agent.py b/toolium/test/utils/ai_utils/test_ai_agent.py index 5040f899..f0bb056d 100644 --- a/toolium/test/utils/ai_utils/test_ai_agent.py +++ b/toolium/test/utils/ai_utils/test_ai_agent.py @@ -68,7 +68,9 @@ def tv_recommendations(user_question): # noqa: ARG001 @pytest.mark.skipif(not os.getenv('AZURE_OPENAI_API_KEY'), reason='AZURE_OPENAI_API_KEY environment variable not set') def test_react_agent(): - agent = create_react_agent(TV_CONTENT_SYSTEM_MESSAGE, tool_method=tv_recommendations, model_name='gpt-4o-mini') + agent = create_react_agent( + TV_CONTENT_SYSTEM_MESSAGE, tool_method=tv_recommendations, provider='azure', model_name='gpt-4o-mini' + ) agent_results = execute_agent(agent) # Check if the agent's final response contains a valid JSON with the expected structure and analyze the result diff --git a/toolium/utils/ai_utils/ai_agent.py b/toolium/utils/ai_utils/ai_agent.py index 562ec46c..032e2b03 100644 --- a/toolium/utils/ai_utils/ai_agent.py +++ b/toolium/utils/ai_utils/ai_agent.py @@ -21,49 +21,47 @@ try: from langchain_core.messages import SystemMessage from langchain_core.tools import Tool - from langchain_openai import AzureChatOpenAI + from langchain_openai import AzureChatOpenAI, ChatOpenAI from langgraph.graph import END, START, MessagesState, StateGraph from langgraph.prebuilt import ToolNode, tools_condition + + AI_IMPORTS = True except ImportError: - AzureChatOpenAI = None + AI_IMPORTS = False from toolium.driver_wrappers_pool import DriverWrappersPool logger = logging.getLogger(__name__) -def create_react_agent(system_message, tool_method, tool_description=None, model_name=None): +def create_react_agent(system_message, tool_method, tool_description=None, provider=None, model_name=None, **kwargs): """ Creates a ReAct agent using the provided system message, tool method and model name. :param system_message: The system message to set the behavior of the assistant :param tool_method: The method that the agent can use as a tool :param tool_description: Optional custom description for the tool. If not provided, uses the method's docstring + :param provider: The AI provider to use (optional, 'azure' or 'openai') :param model_name: The name of the model to use (optional) + :param kwargs: additional parameters to be passed to the LLM chat client :returns: A compiled ReAct agent graph """ - if AzureChatOpenAI is None: + if not AI_IMPORTS: raise ImportError( - "AzureChatOpenAI is not installed. Please run 'pip install toolium[ai]' to use langgraph features", + "AI dependencies are not installed. Please run 'pip install toolium[ai]' to use langgraph features", ) # Define LLM with bound tools - config = DriverWrappersPool.get_default_wrapper().config - model_name = model_name or config.get_optional('AI', 'openai_model', 'gpt-4o-mini') - llm = AzureChatOpenAI(model=model_name) - - # Create tools with custom description if provided + llm = get_llm_chat(provider=provider, model_name=model_name, **kwargs) if tool_description: tools = [Tool(name=tool_method.__name__, description=tool_description, func=tool_method)] else: tools = [tool_method] - llm_with_tools = llm.bind_tools(tools) - # System message + # Define assistant with system message sys_msg = SystemMessage(content=system_message) - # Node def assistant(state: MessagesState): return {'messages': [llm_with_tools.invoke([sys_msg] + state['messages'])]} @@ -85,6 +83,22 @@ def assistant(state: MessagesState): return graph +def get_llm_chat(provider=None, model_name=None, **kwargs): + """ + Get LLM Chat instance based on the provider and model name specified in the parameters or in the configuration file. + + :param provider: the AI provider to use (optional, 'azure' or 'openai') + :param model_name: name of the model to use + :param kwargs: additional parameters to be passed to the chat client + :returns: langchain LLM Chat instance + """ + config = DriverWrappersPool.get_default_wrapper().config + provider = provider or config.get_optional('AI', 'provider', 'openai') + model_name = model_name or config.get_optional('AI', 'openai_model', 'gpt-4o-mini') + llm = AzureChatOpenAI(model=model_name, **kwargs) if provider == 'azure' else ChatOpenAI(model=model_name, **kwargs) + return llm + + def execute_agent(ai_agent, previous_messages=None): """ Executes the given AI agent and logs all conversation messages and tool calls. From 838b36e69af157c86bbfc35490fa27f897b4a6cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Gonz=C3=A1lez=20Alonso?= Date: Fri, 20 Feb 2026 12:48:45 +0100 Subject: [PATCH 5/5] fix unittest --- toolium/test/pageelements/test_page_element.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/toolium/test/pageelements/test_page_element.py b/toolium/test/pageelements/test_page_element.py index 488aa418..62406867 100644 --- a/toolium/test/pageelements/test_page_element.py +++ b/toolium/test/pageelements/test_page_element.py @@ -495,7 +495,6 @@ def test_android_automatic_context_selection_already_in_desired_webview_context_ driver_wrapper.driver.switch_to.window.assert_called_once_with('1234567890') -@pytest.mark.skip(reason='Test disabled temporarily, needs to be reviewed') def test_ios_automatic_context_selection_already_in_desired_webview_context(driver_wrapper): driver_wrapper.is_android_test = mock.MagicMock(return_value=False) driver_wrapper.is_ios_test = mock.MagicMock(return_value=True) @@ -505,8 +504,7 @@ def test_ios_automatic_context_selection_already_in_desired_webview_context(driv driver_wrapper.driver.context = 'WEBVIEW_12345.1' driver_wrapper.driver.execute_script.return_value = [ {'bundleId': 'test.package.fake', 'id': 'WEBVIEW_12345.1'}, - {'bundleId': 'test.package.fake', 'id': 'WEBVIEW_12345.7'}, - {'bundleId': 'test.package.fake', 'id': 'WEBVIEW_54321.1'}, + {'bundleId': 'other.package.fake', 'id': 'WEBVIEW_12345.7'}, ] RegisterPageObject(driver_wrapper).element_webview.web_element # noqa: B018 driver_wrapper.driver.switch_to.context.assert_not_called()