From 313e2e51d1a1a22fa5e3d0f8b36b3321d1e5bcc6 Mon Sep 17 00:00:00 2001 From: mateuszlampert Date: Tue, 10 Feb 2026 20:53:52 +0100 Subject: [PATCH 1/7] chore: add reference files for all functionalities --- .../references/core-utilities.md | 327 ++++++++++++++++++ .../references/reference-audio.md | 311 +++++++++++++++++ .../references/reference-cv-2.md | 190 ++++++++++ .../references/reference-cv.md | 215 ++++++++++++ .../references/reference-llms.md | 259 ++++++++++++++ .../references/reference-models.md | 265 ++++++++++++++ .../references/reference-nlp.md | 169 +++++++++ .../references/reference-ocr.md | 171 +++++++++ 8 files changed, 1907 insertions(+) create mode 100644 skills/react-native-executorch/references/core-utilities.md create mode 100644 skills/react-native-executorch/references/reference-audio.md create mode 100644 skills/react-native-executorch/references/reference-cv-2.md create mode 100644 skills/react-native-executorch/references/reference-cv.md create mode 100644 skills/react-native-executorch/references/reference-llms.md create mode 100644 skills/react-native-executorch/references/reference-models.md create mode 100644 skills/react-native-executorch/references/reference-nlp.md create mode 100644 skills/react-native-executorch/references/reference-ocr.md diff --git a/skills/react-native-executorch/references/core-utilities.md b/skills/react-native-executorch/references/core-utilities.md new file mode 100644 index 000000000..9b5a0c18b --- /dev/null +++ b/skills/react-native-executorch/references/core-utilities.md @@ -0,0 +1,327 @@ +--- +title: RN Executorch core Utilities +description: Reference for using core RN Executorch utils - low-level ExecuTorch bindings, resource management, and error handling. +--- + +# useExecutorchModule + +**Purpose:** Low-level bindings to ExecuTorch Module API for custom model integration. + +**Use cases:** Custom models without dedicated hooks, advanced model control, experimental models, research applications. + +**Important:** Use dedicated hooks (useLLM, useClassification, etc.) when available. This hook is for custom models where no pre-built solution exists. + +## Basic Usage + +```typescript +import { useExecutorchModule } from 'react-native-executorch'; + +const executorchModule = useExecutorchModule({ + modelSource: require('../assets/models/model.pte'), +}); +``` + +## Understanding TensorPtr + +A `TensorPtr` is the JavaScript representation of a tensor passed to the model: + +```typescript +interface TensorPtr { + dataPtr: ArrayBuffer | TypedArray; // Raw data buffer + sizes: number[]; // Tensor shape [batch, channels, height, width] + scalarType: ScalarType; // Data type (FLOAT, INT, etc.) +} +``` + +## Example usage + +```typescript +import { + useExecutorchModule, + ScalarType, + STYLE_TRANSFER_CANDY, +} from 'react-native-executorch'; + +const executorchModule = useExecutorchModule({ + modelSource: STYLE_TRANSFER_CANDY, +}); + +const runInference = async () => { + // Prepare input tensor (example: 640x640 RGB image) + const inputTensor = { + dataPtr: new Float32Array(1 * 3 * 640 * 640), + sizes: [1, 3, 640, 640], + scalarType: ScalarType.FLOAT, + }; + + try { + // Perform the forward operation and receive the stylized image output. + const output = await executorchModule.forward([inputTensor]); + // Interpret the output ArrayBuffer + // foo(output[0].dataPtr); + } catch (error) { + // Log any errors that occur during the forward pass. + console.error('Error during model execution:', error); + } +}; +``` + +## Troubleshooting + +**Preprocessing required:** You must handle all preprocessing (normalization, resizing, color space conversion) yourself. +**Postprocessing required:** Output interpretation is your responsibility based on your model's architecture. +**Shape matching:** Input tensor shapes must exactly match your model's expected input dimensions. +**Use dedicated hooks:** If a hook exists for your use case, use it instead for automatic pre/post-processing. + +## Additional references + +- [useExecutorchModule docs](https://docs.swmansion.com/react-native-executorch/docs/hooks/executorch-bindings/useExecutorchModule) +- [useExecutorchModule API reference](https://docs.swmansion.com/react-native-executorch/docs/api-reference/functions/useExecutorchModule) +- [ExecuTorch Module API](https://pytorch.org/executorch/stable/extension-module.html) +- [Typescript API implementation of useExecutorchModule](https://docs.swmansion.com/react-native-executorch/docs/typescript-api/executorch-bindings/ExecutorchModule) + +--- + +# ResourceFetcher + +**Purpose:** Manage model and resource downloads with pause/resume capabilities. + +**Use cases:** Download management, storage cleanup, progress tracking, offline-first apps. + +## Basic Usage + +```typescript +import { ResourceFetcher } from 'react-native-executorch'; + +// Download multiple resources with progress tracking +const downloadModels = async () => { + try { + const uris = await ResourceFetcher.fetch( + (progress) => + console.log(`Download progress: ${(progress * 100).toFixed(1)}%`), + 'https://example.com/llama3_2.pte', + 'https://example.com/qwen3.pte' + ); + + if (uris) { + console.log('Downloaded files:', uris); + } else { + console.log('Download was paused or cancelled'); + } + } catch (error) { + console.error('Download failed:', error); + } +}; +``` + +## Pause and Resume Downloads + +```typescript +import { ResourceFetcher } from 'react-native-executorch'; + +const uris = ResourceFetcher.fetch( + (progress) => console.log('Total progress:', progress), + 'https://.../llama3_2.pte', + 'https://.../qwen3.pte' +).then((uris) => { + console.log('URI resolved as: ', uris); // since we pause the fetch, uris is resolved to null +}); + +await ResourceFetcher.pauseFetching( + 'https://.../llama3_2.pte', + 'https://.../qwen3.pte' +); + +const resolvedUris = await ResourceFetcher.resumeFetching( + 'https://.../llama3_2.pte', + 'https://.../qwen3.pte' +); +``` + +## Cancel Downloads + +```typescript +import { ResourceFetcher } from 'react-native-executorch'; + +const uris = ResourceFetcher.fetch( + (progress) => console.log('Total progress:', progress), + 'https://.../llama3_2.pte', + 'https://.../qwen3.pte' +).then((uris) => { + console.log('URI resolved as: ', uris); // since we cancel the fetch, uris is resolved to null +}); + +await ResourceFetcher.cancelFetching( + 'https://.../llama3_2.pte', + 'https://.../qwen3.pte' +); +``` + +## Manage Downloaded Resources + +```typescript +import { ResourceFetcher } from 'react-native-executorch'; + +// List all downloaded files +const listFiles = async () => { + const files = await ResourceFetcher.listDownloadedFiles(); + console.log('All downloaded files:', files); + + const models = await ResourceFetcher.listDownloadedModels(); + console.log('Model files:', models); +}; + +// Clean up old resources +const cleanup = async () => { + const oldModelUrl = 'https://example.com/old_model.pte'; + + await ResourceFetcher.deleteResources(oldModelUrl); + console.log('Old model deleted'); +}; +``` + +## Resource Types + +Resources can be: + +- Remote URLs (https://) +- Local file paths (file://) +- Asset references (require()) +- JSON objects + +## Troubleshooting + +**Resume vs re-fetch:** Use `resumeFetching()` for faster resume. Calling `fetch()` again works but is slower. +**Progress callback:** Progress is reported as 0-1 for all downloads combined. +**Null return:** If `fetch()` returns `null`, download was paused or cancelled. +**Network errors:** Implement retry logic with exponential backoff for reliability. +**Storage location:** Downloaded files are stored in application's document directory under `react-native-executorch/` + +## Additional references + +- [ResourceFetcher full reference docs](https://docs.swmansion.com/react-native-executorch/docs/utilities/resource-fetcher) +- [Loading Models guide](https://docs.swmansion.com/react-native-executorch/docs/fundamentals/loading-models) + +--- + +# Error Handling + +**Purpose:** Comprehensive error handling with typed error codes. + +**Use cases:** Debugging, production error recovery, user feedback, logging and monitoring. + +## Basic Error Handling + +```typescript +import { + LLMModule, + LLAMA3_2_1B_QLORA, + RnExecutorchError, + RnExecutorchErrorCode, +} from 'react-native-executorch'; + +const llm = new LLMModule({ + tokenCallback: (token) => console.log(token), + messageHistoryCallback: (messages) => console.log(messages), +}); + +try { + await llm.load(LLAMA3_2_1B_QLORA, (progress) => console.log(progress)); + await llm.sendMessage('Hello!'); +} catch (err) { + if (err instanceof RnExecutorchError) { + console.error(`Error code: ${err.code}`); + console.error(`Error message: ${err.message}`); + } else { + throw err; + } +} +``` + +## Handling Specific Error Types + +```typescript +import { + RnExecutorchError, + RnExecutorchErrorCode, +} from 'react-native-executorch'; + +const handleModelError = async (llm, message: string) => { + try { + await llm.sendMessage(message); + } catch (err) { + if (err instanceof RnExecutorchError) { + switch (err.code) { + case RnExecutorchErrorCode.ModuleNotLoaded: + console.error('Model not loaded. Loading now...'); + await llm.load(LLAMA3_2_1B_QLORA); + // Retry the message + await llm.sendMessage(message); + break; + + case RnExecutorchErrorCode.ModelGenerating: + console.error('Model busy. Waiting...'); + // Wait and retry, or queue the message + break; + + case RnExecutorchErrorCode.InvalidConfig: + console.error('Invalid configuration:', err.message); + // Reset to default config + await llm.configure({ topp: 0.9, temperature: 0.7 }); + break; + + default: + console.error('Unexpected error:', err.message); + throw err; + } + } + } +}; +``` + +## Error Categories + +**Module State Errors** + +- `ModuleNotLoaded` - Model not loaded yet +- `ModelGenerating` - Model already processing + +**Configuration Errors** + +- `InvalidConfig` - Invalid parameters +- `InvalidUserInput` - Bad input data +- `InvalidModelSource` - Wrong model source type +- `WrongDimensions` - Incorrect tensor shape + +**File Operations** + +- `FileReadFailed` - Can't read file +- `FileWriteFailed` - Can't write file + +**Download & Resources** + +- `DownloadInterrupted` - Download didn't complete +- `ResourceFetcherDownloadFailed` - Network/server error +- `ResourceFetcherDownloadInProgress` - Already downloading +- `ResourceFetcherAlreadyPaused` - Already paused +- `ResourceFetcherNotActive` - No active download + +**Runtime Errors** + +- `MemoryAllocationFailed` - Out of memory +- `NotSupported` - Operation not supported +- `InvalidProgram` - Invalid model file + +For complete error reference, see the [Error Handling documentation](https://docs.swmansion.com/react-native-executorch/docs/utilities/error-handling). + +## Troubleshooting + +**Always check instance:** Use `instanceof RnExecutorchError` before accessing `.code`. +**Log error codes:** Include error codes in logs for easier debugging. +**Retry logic:** Implement exponential backoff for network and resource errors. +**User feedback:** Translate error codes into user-friendly messages. + +## Additional references + +- [Error Handling docs](https://docs.swmansion.com/react-native-executorch/docs/utilities/error-handling) +- [Complete error code list](https://docs.swmansion.com/react-native-executorch/docs/utilities/error-handling#reference) diff --git a/skills/react-native-executorch/references/reference-audio.md b/skills/react-native-executorch/references/reference-audio.md new file mode 100644 index 000000000..50bb60c13 --- /dev/null +++ b/skills/react-native-executorch/references/reference-audio.md @@ -0,0 +1,311 @@ +--- +title: Audio related models usage +description: Reference for using Speech to Text, Text to Speech and Voice Activity Detection models. +--- + +# useSpeechToText + +**Purpose:** Convert spoken audio to text (transcription). +**Use cases:** Voice assistants, transcription apps, voice commands, accessibility features. + +## Basic Usage + +```typescript +import { useSpeechToText, WHISPER_TINY_EN } from 'react-native-executorch'; +import { AudioContext } from 'react-native-audio-api'; + +const model = useSpeechToText({ model: WHISPER_TINY_EN }); + +// Process audio file +const audioContext = new AudioContext({ sampleRate: 16000 }); +const decodedAudio = await audioContext.decodeAudioDataSource(audioUri); +const waveform = decodedAudio.getChannelData(0); + +const transcription = await model.transcribe(waveform); +console.log(transcription); +``` + +## Multilingual Transcription + +```typescript +import { WHISPER_TINY } from 'react-native-executorch'; + +const model = useSpeechToText({ model: WHISPER_TINY }); + +// Specify language +const transcription = await model.transcribe(spanishAudio, { + language: 'es', +}); +``` + +## Streaming Transcription + +```typescript +import { AudioRecorder, AudioManager } from 'react-native-audio-api'; + +const recorder = new AudioRecorder({ + sampleRate: 16000, + bufferLengthInSamples: 1600, +}); + +// Start streaming +recorder.onAudioReady(({ buffer }) => { + model.streamInsert(buffer.getChannelData(0)); +}); +recorder.start(); + +await model.stream(); + +// Access results +console.log(model.committedTranscription); +console.log(model.nonCommittedTranscription); + +// Stop streaming +recorder.stop(); +model.streamStop(); +``` + +## Troubleshooting + +**Audio must be 16kHz:** Ensure proper sample rate before processing +**Streaming algorithm:** Use whisper-streaming for longer audio (handles 30s chunks automatically) + +## Additional references + +- [useSpeechToText docs](https://docs.swmansion.com/react-native-executorch/docs/api-reference/functions/useSpeechToText) +- [HuggingFace STT collection](https://huggingface.co/collections/software-mansion/speech-to-text) +- [Available models](https://docs.swmansion.com/react-native-executorch/docs/api-reference#models---speech-to-text) +- [useSpeechToText API reference](https://docs.swmansion.com/react-native-executorch/docs/api-reference/functions/useSpeechToText) +- [Typescript API implementation of useSpeechToText](https://docs.swmansion.com/react-native-executorch/docs/typescript-api/natural-language-processing/SpeechToTextModule) + +--- + +# useTextToSpeech + +**Purpose:** Convert text to natural-sounding speech (TTS). +**Use cases:** Voice assistants, audiobooks, accessibility tools, voice navigation. + +## Basic Usage + +```typescript +import { + useTextToSpeech, + KOKORO_MEDIUM, + KOKORO_VOICE_AF_HEART, +} from 'react-native-executorch'; +import { AudioContext } from 'react-native-audio-api'; + +const model = useTextToSpeech({ + model: KOKORO_MEDIUM, + voice: KOKORO_VOICE_AF_HEART, +}); + +const audioContext = new AudioContext({ sampleRate: 24000 }); + +const handleSpeech = async (text: string) => { + const speed = 1.0; + const waveform = await model.forward(text, speed); + + const audioBuffer = audioContext.createBuffer(1, waveform.length, 24000); + audioBuffer.getChannelData(0).set(waveform); + + const source = audioContext.createBufferSource(); + source.buffer = audioBuffer; + source.connect(audioContext.destination); + source.start(); +}; +``` + +## Streaming TTS + +```typescript +// Stream chunks for lower latency +await tts.stream({ + text: 'Long text to be streamed chunk by chunk...', + speed: 1.0, + onNext: async (chunk) => { + return new Promise((resolve) => { + const buffer = ctx.createBuffer(1, chunk.length, 24000); + buffer.getChannelData(0).set(chunk); + + const source = ctx.createBufferSource(); + source.buffer = buffer; + source.connect(ctx.destination); + source.onEnded = () => resolve(); + source.start(); + }); + }, +}); +``` + +## Available Models & Voices + +**Model:** Kokoro (English only) + +For all available models check out [this exported HuggingFace models collection](https://huggingface.co/software-mansion/react-native-executorch-kokoro). + +**Available Voices:** + +- `KOKORO_VOICE_AF_HEART` - Female, heart +- `KOKORO_VOICE_AF_SKY` - Female, sky +- `KOKORO_VOICE_AF_BELLA` - Female, bella +- `KOKORO_VOICE_AF_NICOLE` - Female, nicole +- `KOKORO_VOICE_AF_SARAH` - Female, sarah +- `KOKORO_VOICE_AM_ADAM` - Male, adam +- `KOKORO_VOICE_AM_MICHAEL` - Male, michael +- `KOKORO_VOICE_BF_EMMA` - British Female, emma +- `KOKORO_VOICE_BF_ISABELLA` - British Female, isabella +- `KOKORO_VOICE_BM_GEORGE` - British Male, george +- `KOKORO_VOICE_BM_LEWIS` - British Male, lewis + +## Troubleshooting + +**Streaming vs Forward:** Use `stream()` for long texts to reduce time-to-first-audio + +## Additional references + +- [useTextToSpeech docs - reference and examples](https://docs.swmansion.com/react-native-executorch/docs/api-reference/functions/useTextToSpeech) +- [Supported Voices](https://docs.swmansion.com/react-native-executorch/docs/api-reference#tts-supported-voices) +- [useTextToSpeech API reference](https://docs.swmansion.com/react-native-executorch/docs/api-reference/functions/useTextToSpeech) +- [HuggingFace TTS collection](https://huggingface.co/collections/software-mansion/text-to-speech) +- [Typescript API implementation of useTextToSpeech hook](https://docs.swmansion.com/react-native-executorch/docs/typescript-api/natural-language-processing/TextToSpeechModule) + +--- + +# useVAD + +**Purpose:** Detect speech segments in audio (Voice Activity Detection). + +**Use cases:** Audio preprocessing, removing silence, speech segmentation, smart recording. + +## Basic Usage + +```typescript +import { useVAD, FSMN_VAD } from 'react-native-executorch'; +import { AudioContext } from 'react-native-audio-api'; +import * as FileSystem from 'expo-file-system'; + +const model = useVAD({ + model: FSMN_VAD, +}); + +const { uri } = await FileSystem.downloadAsync( + 'https://some-audio-url.com/file.mp3', + FileSystem.cacheDirectory + 'audio_file' +); + +const audioContext = new AudioContext({ sampleRate: 16000 }); +const decodedAudioData = await audioContext.decodeAudioDataSource(uri); +const audioBuffer = decodedAudioData.getChannelData(0); + +try { + // NOTE: to obtain segments in seconds, you need to divide + // start / end of the segment by the sampling rate (16k) + + const speechSegments = await model.forward(audioBuffer); + console.log(speechSegments); +} catch (error) { + console.error('Error during running VAD model', error); +} +``` + +## Example usage + +```tsx +import React from 'react'; +import { Button, Text, SafeAreaView } from 'react-native'; +import { useVAD, FSMN_VAD } from 'react-native-executorch'; +import { AudioContext } from 'react-native-audio-api'; +import * as FileSystem from 'expo-file-system'; + +export default function App() { + const model = useVAD({ + model: FSMN_VAD, + }); + + const audioURL = 'https://some-audio-url.com/file.mp3'; + + const handleAudio = async () => { + if (!model) { + console.error('VAD model is not loaded yet.'); + return; + } + + console.log('Processing URL:', audioURL); + + try { + const { uri } = await FileSystem.downloadAsync( + audioURL, + FileSystem.cacheDirectory + 'vad_example.tmp' + ); + + const audioContext = new AudioContext({ sampleRate: 16000 }); + const originalDecodedBuffer = + await audioContext.decodeAudioDataSource(uri); + const originalChannelData = originalDecodedBuffer.getChannelData(0); + + const segments = await model.forward(originalChannelData); + if (segments.length === 0) { + console.log('No speech segments were found.'); + return; + } + console.log(`Found ${segments.length} speech segments.`); + + const totalLength = segments.reduce( + (sum, seg) => sum + (seg.end - seg.start), + 0 + ); + const newAudioBuffer = audioContext.createBuffer( + 1, // Mono + totalLength, + originalDecodedBuffer.sampleRate + ); + const newChannelData = newAudioBuffer.getChannelData(0); + + let offset = 0; + for (const segment of segments) { + const slice = originalChannelData.subarray(segment.start, segment.end); + newChannelData.set(slice, offset); + offset += slice.length; + } + + // Play the processed audio + const source = audioContext.createBufferSource(); + source.buffer = newAudioBuffer; + source.connect(audioContext.destination); + source.start(); + } catch (error) { + console.error('Error processing audio data:', error); + } + }; + + return ( + + + Press the button to process and play speech from a sample file. + +