Skip to content

chore: Add basic statistics to demo apps #985

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
msluszniak wants to merge 7 commits into
base: main
Choose a base branch
from
+335 −47
Open

chore: Add basic statistics to demo apps #985

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
1b656cf
chore: Add basic statistics to demo apps
msluszniak Mar 19, 2026
cbabf51
chore: handle rebased, magled code of app in text to image
msluszniak Mar 20, 2026
a6d5440
Fix T2S streaming on unfinished sentences
IgorSwat Mar 20, 2026
7bc532e
Apply suggestions from code review
msluszniak Mar 21, 2026
64d4f44
chore: add model selection and basic statistics to instance segmentation
msluszniak Mar 21, 2026
95ed309
chore: split inference time in image embeddings example
msluszniak Mar 21, 2026
55c912f
chore: reformat
msluszniak Mar 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions apps/computer-vision/app/classification/index.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,14 @@ import { BottomBar } from '../../components/BottomBar';
import React, { useContext, useEffect, useState } from 'react';
import { GeneratingContext } from '../../context';
import ScreenWrapper from '../../ScreenWrapper';
import { StatsBar } from '../../components/StatsBar';

export default function ClassificationScreen() {
const [results, setResults] = useState<{ label: string; score: number }[]>(
[]
);
const [imageUri, setImageUri] = useState('');
const [inferenceTime, setInferenceTime] = useState<number | null>(null);

const model = useClassification({ model: EFFICIENTNET_V2_S_QUANTIZED });
const { setGlobalGenerating } = useContext(GeneratingContext);
Expand All @@ -34,7 +36,9 @@ export default function ClassificationScreen() {
const runForward = async () => {
if (imageUri) {
try {
const start = Date.now();
const output = await model.forward(imageUri);
setInferenceTime(Date.now() - start);
const top10 = Object.entries(output)
.sort(([, a], [, b]) => (b as number) - (a as number))
.slice(0, 10)
Expand Down Expand Up @@ -80,6 +84,7 @@ export default function ClassificationScreen() {
</View>
)}
</View>
<StatsBar inferenceTime={inferenceTime} />
<BottomBar
handleCameraPress={handleCameraPress}
runForward={runForward}
Expand Down
48 changes: 46 additions & 2 deletions apps/computer-vision/app/instance_segmentation/index.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,17 @@
import Spinner from '../../components/Spinner';
import { BottomBar } from '../../components/BottomBar';
import { getImage } from '../../utils';
import { useInstanceSegmentation, YOLO26N_SEG } from 'react-native-executorch';
import { ModelPicker, ModelOption } from '../../components/ModelPicker';
import {
useInstanceSegmentation,
YOLO26N_SEG,
YOLO26S_SEG,
YOLO26M_SEG,
YOLO26L_SEG,
YOLO26X_SEG,
RF_DETR_NANO_SEG,
InstanceSegmentationModelSources,
} from 'react-native-executorch';
import {
View,
StyleSheet,
Expand All @@ -16,8 +26,22 @@ import ImageWithMasks, {
buildDisplayInstances,
DisplayInstance,
} from '../../components/ImageWithMasks';
import { StatsBar } from '../../components/StatsBar';

const MODELS: ModelOption<InstanceSegmentationModelSources>[] = [
{ label: 'Yolo26N', value: YOLO26N_SEG },
{ label: 'Yolo26S', value: YOLO26S_SEG },
{ label: 'Yolo26M', value: YOLO26M_SEG },
{ label: 'Yolo26L', value: YOLO26L_SEG },
{ label: 'Yolo26X', value: YOLO26X_SEG },
{ label: 'RF-DeTR Nano', value: RF_DETR_NANO_SEG },
];

export default function InstanceSegmentationScreen() {
const [selectedModel, setSelectedModel] =
useState<InstanceSegmentationModelSources>(YOLO26N_SEG);
const [inferenceTime, setInferenceTime] = useState<number | null>(null);

const { setGlobalGenerating } = useContext(GeneratingContext);

const {
Expand All @@ -28,7 +52,7 @@ export default function InstanceSegmentationScreen() {
error,
getAvailableInputSizes,
} = useInstanceSegmentation({
model: YOLO26N_SEG,
model: selectedModel,
});

const [imageUri, setImageUri] = useState('');
Expand Down Expand Up @@ -60,12 +84,14 @@ export default function InstanceSegmentationScreen() {
height: image.height ?? 0,
});
setInstances([]);
setInferenceTime(null);
Copy link
Contributor

@NorbertKlockiewicz NorbertKlockiewicz Mar 23, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is missing in the rest of the computer vision screens, so once we switch the model in those the stale inference time still will be visible.

};

const runForward = async () => {
if (!imageUri || imageSize.width === 0 || imageSize.height === 0) return;

try {
const start = Date.now();
const output = await forward(imageUri, {
confidenceThreshold: 0.5,
iouThreshold: 0.55,
Expand All @@ -74,6 +100,8 @@ export default function InstanceSegmentationScreen() {
inputSize: selectedInputSize ?? undefined,
});

setInferenceTime(Date.now() - start);

// Convert raw masks → small Skia images immediately.
// Raw Uint8Array mask buffers (backed by native OwningArrayBuffer)
// go out of scope here and become eligible for GC right away.
Expand Down Expand Up @@ -168,6 +196,22 @@ export default function InstanceSegmentationScreen() {
)}
</View>

<ModelPicker
models={MODELS}
selectedModel={selectedModel}
disabled={isGenerating}
onSelect={(m) => {
setSelectedModel(m);
setInstances([]);
setInferenceTime(null);
}}
/>

<StatsBar
inferenceTime={inferenceTime}
detectionCount={instances.length > 0 ? instances.length : null}
/>

<BottomBar
handleCameraPress={handleCameraPress}
runForward={runForward}
Expand Down
8 changes: 8 additions & 0 deletions apps/computer-vision/app/object_detection/index.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import ImageWithBboxes from '../../components/ImageWithBboxes';
import React, { useContext, useEffect, useState } from 'react';
import { GeneratingContext } from '../../context';
import ScreenWrapper from '../../ScreenWrapper';
import { StatsBar } from '../../components/StatsBar';

const MODELS: ModelOption<ObjectDetectionModelSources>[] = [
{ label: 'RF-DeTR Nano', value: RF_DETR_NANO },
Expand All @@ -29,6 +30,7 @@ export default function ObjectDetectionScreen() {
}>();
const [selectedModel, setSelectedModel] =
useState<ObjectDetectionModelSources>(RF_DETR_NANO);
const [inferenceTime, setInferenceTime] = useState<number | null>(null);

const model = useObjectDetection({ model: selectedModel });
const { setGlobalGenerating } = useContext(GeneratingContext);
Expand All @@ -52,7 +54,9 @@ export default function ObjectDetectionScreen() {
const runForward = async () => {
if (imageUri) {
try {
const start = Date.now();
const output = await model.forward(imageUri);
setInferenceTime(Date.now() - start);
setResults(output);
} catch (e) {
console.error(e);
Expand Down Expand Up @@ -100,6 +104,10 @@ export default function ObjectDetectionScreen() {
setResults([]);
}}
/>
<StatsBar
inferenceTime={inferenceTime}
detectionCount={results.length > 0 ? results.length : null}
/>
<BottomBar
handleCameraPress={handleCameraPress}
runForward={runForward}
Expand Down
8 changes: 8 additions & 0 deletions apps/computer-vision/app/ocr/index.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import ImageWithBboxes2 from '../../components/ImageWithOCRBboxes';
import React, { useContext, useEffect, useState } from 'react';
import { GeneratingContext } from '../../context';
import ScreenWrapper from '../../ScreenWrapper';
import { StatsBar } from '../../components/StatsBar';

type OCRModelSources = OCRProps['model'];

Expand All @@ -40,6 +41,7 @@ export default function OCRScreen() {
}>();
const [selectedModel, setSelectedModel] =
useState<OCRModelSources>(OCR_ENGLISH);
const [inferenceTime, setInferenceTime] = useState<number | null>(null);

const model = useOCR({
model: selectedModel,
Expand All @@ -63,7 +65,9 @@ export default function OCRScreen() {

const runForward = async () => {
try {
const start = Date.now();
const output = await model.forward(imageUri);
setInferenceTime(Date.now() - start);
setResults(output);
} catch (e) {
console.error(e);
Expand Down Expand Up @@ -123,6 +127,10 @@ export default function OCRScreen() {
setResults([]);
}}
/>
<StatsBar
inferenceTime={inferenceTime}
detectionCount={results.length > 0 ? results.length : null}
/>
<BottomBar
handleCameraPress={handleCameraPress}
runForward={runForward}
Expand Down
8 changes: 8 additions & 0 deletions apps/computer-vision/app/ocr_vertical/index.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import ImageWithBboxes2 from '../../components/ImageWithOCRBboxes';
import React, { useContext, useEffect, useState } from 'react';
import { GeneratingContext } from '../../context';
import ScreenWrapper from '../../ScreenWrapper';
import { StatsBar } from '../../components/StatsBar';

export default function VerticalOCRScree() {
const [imageUri, setImageUri] = useState('');
Expand All @@ -15,6 +16,7 @@ export default function VerticalOCRScree() {
width: number;
height: number;
}>();
const [inferenceTime, setInferenceTime] = useState<number | null>(null);
const model = useVerticalOCR({
model: OCR_ENGLISH,
independentCharacters: true,
Expand All @@ -38,7 +40,9 @@ export default function VerticalOCRScree() {

const runForward = async () => {
try {
const start = Date.now();
const output = await model.forward(imageUri);
setInferenceTime(Date.now() - start);
setResults(output);
} catch (e) {
console.error(e);
Expand Down Expand Up @@ -89,6 +93,10 @@ export default function VerticalOCRScree() {
</View>
)}
</View>
<StatsBar
inferenceTime={inferenceTime}
detectionCount={results.length > 0 ? results.length : null}
/>
<BottomBar
handleCameraPress={handleCameraPress}
runForward={runForward}
Expand Down
5 changes: 5 additions & 0 deletions apps/computer-vision/app/semantic_segmentation/index.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import { View, StyleSheet, Image } from 'react-native';
import React, { useContext, useEffect, useState } from 'react';
import { GeneratingContext } from '../../context';
import ScreenWrapper from '../../ScreenWrapper';
import { StatsBar } from '../../components/StatsBar';

const numberToColor: number[][] = [
[255, 87, 51], // 0 Red
Expand Down Expand Up @@ -75,6 +76,7 @@ export default function SemanticSegmentationScreen() {
const [imageSize, setImageSize] = useState({ width: 0, height: 0 });
const [segImage, setSegImage] = useState<SkImage | null>(null);
const [canvasSize, setCanvasSize] = useState({ width: 0, height: 0 });
const [inferenceTime, setInferenceTime] = useState<number | null>(null);

useEffect(() => {
setGlobalGenerating(isGenerating);
Expand All @@ -91,6 +93,7 @@ export default function SemanticSegmentationScreen() {
const runForward = async () => {
if (!imageUri || imageSize.width === 0 || imageSize.height === 0) return;
try {
const start = Date.now();
const { width, height } = imageSize;
const output = await forward(imageUri, [], true);
const argmax = output.ARGMAX || [];
Expand Down Expand Up @@ -119,6 +122,7 @@ export default function SemanticSegmentationScreen() {
width * 4
);
setSegImage(img);
setInferenceTime(Date.now() - start);
} catch (e) {
console.error(e);
}
Expand Down Expand Up @@ -179,6 +183,7 @@ export default function SemanticSegmentationScreen() {
setSegImage(null);
}}
/>
<StatsBar inferenceTime={inferenceTime} />
<BottomBar
handleCameraPress={handleCameraPress}
runForward={runForward}
Expand Down
5 changes: 5 additions & 0 deletions apps/computer-vision/app/style_transfer/index.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import { View, StyleSheet, Image } from 'react-native';
import React, { useContext, useEffect, useState } from 'react';
import { GeneratingContext } from '../../context';
import ScreenWrapper from '../../ScreenWrapper';
import { StatsBar } from '../../components/StatsBar';

type StyleTransferModelSources = {
modelName: StyleTransferModelName;
Expand All @@ -42,6 +43,7 @@ export default function StyleTransferScreen() {

const [imageUri, setImageUri] = useState('');
const [styledUri, setStyledUri] = useState('');
const [inferenceTime, setInferenceTime] = useState<number | null>(null);

const handleCameraPress = async (isCamera: boolean) => {
const image = await getImage(isCamera);
Expand All @@ -55,7 +57,9 @@ export default function StyleTransferScreen() {
const runForward = async () => {
if (imageUri) {
try {
const start = Date.now();
const uri = await model.forward(imageUri, 'url');
setInferenceTime(Date.now() - start);
setStyledUri(uri);
} catch (e) {
console.error(e);
Expand Down Expand Up @@ -96,6 +100,7 @@ export default function StyleTransferScreen() {
setStyledUri('');
}}
/>
<StatsBar inferenceTime={inferenceTime} />
<BottomBar
handleCameraPress={handleCameraPress}
runForward={runForward}
Expand Down
12 changes: 11 additions & 1 deletion apps/computer-vision/app/text_to_image/index.tsx
View file
Open in desktop
Copy link
Contributor

@barhanc barhanc Mar 23, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The app can still crash if one starts generating without closing the model picker and then tries to switch the model. Additionally it is possible to trigger Error: mutex lock failed: Invalid argument if in the above, one interrupts the model and then quickly tries to switch the model.

Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import { GeneratingContext } from '../../context';
import ColorPalette from '../../colors';
import ProgressBar from '../../components/ProgressBar';
import { Ionicons } from '@expo/vector-icons';
import { StatsBar } from '../../components/StatsBar';

type TextToImageModelSources = TextToImageProps['model'];

Expand All @@ -39,6 +40,7 @@ export default function TextToImageScreen() {
const [selectedModel, setSelectedModel] = useState<TextToImageModelSources>(
BK_SDM_TINY_VPRED_256
);
const [generationTime, setGenerationTime] = useState<number | null>(null);

const imageSize = 224;
const model = useTextToImage({
Expand All @@ -55,8 +57,13 @@ export default function TextToImageScreen() {
const runForward = async () => {
if (!input.trim()) return;
try {
const start = Date.now();
const output = await model.generate(input, imageSize, steps);
if (output.length) setImage(output);

if (output.length) {
setImage(output);
setGenerationTime(Date.now() - start);
}
} catch (e) {
console.error(e);
} finally {
Expand Down Expand Up @@ -105,6 +112,7 @@ export default function TextToImageScreen() {
onSelect={(m) => {
setSelectedModel(m);
setImage(null);
setGenerationTime(null);
}}
/>

Expand All @@ -124,6 +132,8 @@ export default function TextToImageScreen() {
</TouchableOpacity>
</View>

<StatsBar inferenceTime={generationTime} />

<View style={styles.inputRow}>
<TextInput
style={styles.textInput}
Expand Down
Loading
Loading