feat: dynamic UI controlled by the LLM with @react-native-ai/json-ui, tool call visualization

apps/expo-example/package.json

@@ -40,6 +40,7 @@
     "expo-symbols": "~1.0.8",
     "expo-system-ui": "^6.0.9",
     "jotai": "^2.12.2",
+    "@react-native-ai/json-ui": "workspace:*",
     "llama.rn": "^0.10.1",
     "react": "19.1.0",
     "react-native": "0.81.5",

apps/expo-example/src/components/adapters/appleSetupAdapter.ts

@@ -1,15 +1,13 @@
 import type { LanguageModelV3, SpeechModelV3 } from '@ai-sdk/provider'
+import type { Tool } from '@ai-sdk/provider-utils'
 import { apple, createAppleProvider } from '@react-native-ai/apple'
-import { ToolSet } from 'ai'
 
 import type { SetupAdapter } from '../../config/providers.common'
 
 export const createAppleLanguageSetupAdapter = (
-  tools: ToolSet = {}
+  tools: Record<string, Tool>
 ): SetupAdapter<LanguageModelV3> => {
-  const apple = createAppleProvider({
-    availableTools: tools,
-  })
+  const apple = createAppleProvider({ availableTools: tools })
   const model = apple.languageModel()
   return {
     model,

apps/expo-example/src/components/adapters/llamaModelSetupAdapter.ts

@@ -1,6 +1,5 @@
 import type { LanguageModelV3 } from '@ai-sdk/provider'
 import { llama } from '@react-native-ai/llama'
-import { ToolSet } from 'ai'
 
 import {
   downloadModel,
@@ -11,8 +10,7 @@ import type { Availability, SetupAdapter } from '../../config/providers.common'
 import { isLlamaModelDownloaded } from '../../utils/llamaStorageUtils'
 
 export const createLlamaLanguageSetupAdapter = (
-  hfModelId: string,
-  tools: ToolSet = {}
+  hfModelId: string
 ): SetupAdapter<LanguageModelV3> => {
   const modelPath = getModelPath(hfModelId)
   const model = llama.languageModel(modelPath, {

apps/expo-example/src/config/providers.common.ts

@@ -3,7 +3,6 @@ import type { LanguageModelV3, SpeechModelV3 } from '@ai-sdk/provider'
 import { createLlamaLanguageSetupAdapter } from '../components/adapters/llamaModelSetupAdapter'
 import { createLlamaSpeechSetupAdapter } from '../components/adapters/llamaSpeechSetupAdapter'
 import { createMLCLanguageSetupAdapter } from '../components/adapters/mlcModelSetupAdapter'
-import { toolDefinitions } from '../tools'
 
 export type Availability = 'yes' | 'no' | 'availableForDownload'
 
@@ -42,8 +41,7 @@ export const commonLanguageAdapters: SetupAdapter<LanguageModelV3>[] = [
     'ggml-org/SmolLM3-3B-GGUF/SmolLM3-Q4_K_M.gguf'
   ),
   createLlamaLanguageSetupAdapter(
-    'Qwen/Qwen2.5-3B-Instruct-GGUF/qwen2.5-3b-instruct-q3_k_m.gguf',
-    toolDefinitions
+    'Qwen/Qwen2.5-3B-Instruct-GGUF/qwen2.5-3b-instruct-q3_k_m.gguf'
   ),
   createMLCLanguageSetupAdapter('Llama-3.2-1B-Instruct'),
   createMLCLanguageSetupAdapter('Llama-3.2-3B-Instruct'),

apps/expo-example/src/screens/ChatScreen/ChatMessageBubble.tsx

@@ -1,37 +1,104 @@
 import React from 'react'
-import { StyleSheet, Text, View } from 'react-native'
+import { Pressable, StyleSheet, Text, View } from 'react-native'
 
 import { colors } from '../../theme/colors'
 
 type ChatMessageBubbleProps = {
   content: string
   isUser: boolean
+  messageType?: 'text' | 'toolExecution'
+  toolExecution?: {
+    toolName: string
+    payload: unknown
+    result?: unknown
+  }
 }
 
-export function ChatMessageBubble({ content, isUser }: ChatMessageBubbleProps) {
+export function ChatMessageBubble({
+  content,
+  isUser,
+  messageType = 'text',
+  toolExecution,
+}: ChatMessageBubbleProps) {
+  const [expanded, setExpanded] = React.useState(false)
+  const isToolHint = messageType === 'toolExecution'
+  const toolLabel = toolExecution
+    ? `Tool executed: ${toolExecution.toolName}`
+    : content
+  const payload = toolExecution?.payload
+    ? JSON.stringify(toolExecution.payload, null, 2)
+    : ''
+  const result =
+    toolExecution?.result !== undefined
+      ? JSON.stringify(toolExecution.result, null, 2)
+      : ''
+  const hasPayload = payload !== '{}' && payload.length > 0
+  const hasResult = result.length > 0
+  const hasDetails = hasPayload || hasResult
+
   return (
     <View
       style={[
         styles.messageRow,
-        isUser ? styles.messageRowUser : styles.messageRowAssistant,
+        isToolHint
+          ? styles.messageRowAssistant
+          : isUser
+            ? styles.messageRowUser
+            : styles.messageRowAssistant,
       ]}
     >
-      <View
+      <Pressable
+        disabled={!isToolHint}
+        onPress={() => hasDetails && setExpanded((prev) => !prev)}
         style={[
           styles.messageBubble,
-          isUser ? styles.messageBubbleUser : styles.messageBubbleAssistant,
+          isToolHint
+            ? styles.messageBubbleToolHint
+            : isUser
+              ? styles.messageBubbleUser
+              : styles.messageBubbleAssistant,
         ]}
       >
-        <Text
-          selectable
-          style={[
-            styles.messageText,
-            isUser ? styles.messageTextUser : styles.messageTextAssistant,
-          ]}
-        >
-          {content}
-        </Text>
-      </View>
+        {isToolHint ? (
+          <View style={styles.toolHintHeader}>
+            <Text
+              selectable
+              style={[styles.messageText, styles.messageTextToolHint]}
+            >
+              {toolLabel}
+            </Text>
+            {hasDetails && (
+              <Text style={styles.expandIcon}>{expanded ? '[-]' : '[+]'}</Text>
+            )}
+          </View>
+        ) : (
+          <Text
+            selectable
+            style={[
+              styles.messageText,
+              isUser ? styles.messageTextUser : styles.messageTextAssistant,
+            ]}
+          >
+            {content}
+          </Text>
+        )}
+        {isToolHint && expanded && hasPayload ? (
+          <View style={styles.detailBlock}>
+            <Text style={styles.detailLabel}>args</Text>
+            <Text selectable style={styles.payloadText}>
+              {payload}
+            </Text>
+          </View>
+        ) : null}
+        {isToolHint && expanded && hasResult ? (
+          <View style={styles.detailBlock}>
+            <Text style={styles.detailLabel}>result</Text>
+            <Text selectable style={styles.payloadText}>
+              {result}
+            </Text>
+          </View>
+        ) : null}
+      </Pressable>
     </View>
   )
 }
@@ -59,6 +126,10 @@ const styles = StyleSheet.create({
   messageBubbleAssistant: {
     backgroundColor: colors.secondarySystemBackground as any,
   },
+  messageBubbleToolHint: {
+    backgroundColor: colors.tertiarySystemFill as any,
+    borderRadius: 14,
+  },
   messageText: {
     fontSize: 16,
     lineHeight: 22,
@@ -69,4 +140,35 @@ const styles = StyleSheet.create({
   messageTextAssistant: {
     color: colors.label as any,
   },
+  messageTextToolHint: {
+    color: colors.secondaryLabel as any,
+    fontSize: 14,
+    lineHeight: 20,
+  },
+  toolHintHeader: {
+    flexDirection: 'row',
+    alignItems: 'center',
+    justifyContent: 'space-between',
+    gap: 8,
+  },
+  expandIcon: {
+    color: colors.tertiaryLabel as any,
+    fontSize: 12,
+  },
+  payloadText: {
+    marginTop: 8,
+    fontFamily: 'Menlo',
+    fontSize: 12,
+    lineHeight: 18,
+    color: colors.tertiaryLabel as any,
+  },
+  detailBlock: {
+    marginTop: 8,
+  },
+  detailLabel: {
+    fontSize: 11,
+    color: colors.tertiaryLabel as any,
+    textTransform: 'uppercase',
+    letterSpacing: 0.5,
+  },
 })

apps/expo-example/src/screens/ChatScreen/ChatMessages.tsx

@@ -1,3 +1,4 @@
+import { GenerativeUIView } from '@react-native-ai/json-ui'
 import React, { useEffect, useRef } from 'react'
 import { ScrollView, StyleSheet, View } from 'react-native'
 import { useKeyboardHandler } from 'react-native-keyboard-controller'
@@ -8,6 +9,7 @@ import Reanimated, {
 import { useSafeAreaInsets } from 'react-native-safe-area-context'
 import { scheduleOnRN } from 'react-native-worklets'
 
+import { getChatUISpecFromChats, useChatStore } from '../../store/chatStore'
 import { ChatEmptyState } from './ChatEmptyState'
 import { ChatInputBar } from './ChatInputBar'
 import { ChatMessageBubble } from './ChatMessageBubble'
@@ -16,6 +18,12 @@ type ChatMessage = {
   id: string
   role: string
   content: string
+  type?: 'text' | 'toolExecution'
+  toolExecution?: {
+    toolName: string
+    payload: unknown
+    result?: unknown
+  }
 }
 
 type ChatMessagesProps = {
@@ -62,6 +70,8 @@ export function ChatMessages({
 
   useEffect(scrollToBottom, [messages.length])
 
+  const { chats, currentChatId } = useChatStore()
+
   return (
     <>
       <ScrollView
@@ -72,7 +82,7 @@ export function ChatMessages({
         {messages.length === 0 ? (
           <ChatEmptyState
             title="What can I help you with?"
-            subtitle={`Start a conversation with ${selectedModelLabel}. Ask questions, get creative, or explore ideas.`}
+            subtitle={`Start a conversation with ${selectedModelLabel}. Ask questions, ask it to add new UI elements to the screen, get creative, or explore ideas.`}
           />
         ) : (
           <View style={styles.messageList}>
@@ -81,8 +91,20 @@ export function ChatMessages({
                 key={message.id}
                 content={message.content}
                 isUser={message.role === 'user'}
+                messageType={message.type}
+                toolExecution={message.toolExecution}
               />
             ))}
+
+            <GenerativeUIView
+              spec={
+                currentChatId
+                  ? getChatUISpecFromChats(chats, currentChatId)
+                  : null
+              }
+              loading={isGenerating}
+              showCollapsibleJSON
+            />
           </View>
         )}
       </ScrollView>