From 36b495326b050c6ceddc8fd3b8d99752d0159379 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Fri, 24 Apr 2026 18:07:54 +0200 Subject: [PATCH] feat: Add `allowPhysicalBufferResizing`, remove `enablePreviewSizedOutputBuffers` --- apps/simple-camera/__tests__/README.md | 5 +- .../visioncamera.constraints.harness.ts | 2 +- .../__tests__/visioncamera.frame.harness.ts | 63 +++++++++++----- docs/content/docs/a-frame.mdx | 26 +++++++ .../src/views/SkiaCamera.tsx | 28 ++++---- .../hybrids/outputs/HybridDepthFrameOutput.kt | 1 - .../hybrids/outputs/HybridFrameOutput.kt | 1 - .../Outputs/HybridCameraFrameOutput.swift | 72 ++++++++++++++++--- .../src/hooks/useFrameOutput.ts | 6 +- .../specs/outputs/CameraFrameOutput.nitro.ts | 39 +++++++--- 10 files changed, 184 insertions(+), 59 deletions(-) diff --git a/apps/simple-camera/__tests__/README.md b/apps/simple-camera/__tests__/README.md index c88d1f924c..3753ce7d22 100644 --- a/apps/simple-camera/__tests__/README.md +++ b/apps/simple-camera/__tests__/README.md @@ -39,7 +39,7 @@ Tests are split by domain. Each file tests one slice of the imperative | [visioncamera.session.harness.ts](visioncamera.session.harness.ts) | `createCameraSession`, `configure`, `start`, `stop`, `addOnStartedListener` / `addOnStoppedListener` / `addOnErrorListener` / interruption listeners, reconfigure-while-running, multi-cam | | [visioncamera.photo.harness.ts](visioncamera.photo.harness.ts) | `createPhotoOutput`, `capturePhoto` / `capturePhotoToFile`, container formats (JPEG, HEIC, DNG), flash / mirror / quality / resolution options, capture lifecycle callbacks, preview images | | [visioncamera.video.harness.ts](visioncamera.video.harness.ts) | `createVideoOutput`, `Recorder` lifecycle, audio, `maxDuration` / `maxFileSize` stops, pause / resume / cancel, persistent recorder, higher-resolution codecs | -| [visioncamera.frame.harness.ts](visioncamera.frame.harness.ts) | `createFrameOutput`, worklet install via `react-native-vision-camera-worklets`, YUV / RGB / native pixel formats, `scheduleOnRN`, `createSynchronizable`, `setOnFrameDroppedCallback`, `enablePreviewSizedOutputBuffers` | +| [visioncamera.frame.harness.ts](visioncamera.frame.harness.ts) | `createFrameOutput`, worklet install via `react-native-vision-camera-worklets`, YUV / RGB / native pixel formats, `scheduleOnRN`, `createSynchronizable`, `setOnFrameDroppedCallback`, `allowPhysicalBufferResizing` | | [visioncamera.constraints.harness.ts](visioncamera.constraints.harness.ts) | `VisionCamera.resolveConstraints` + `onSessionConfigSelected`, FPS / HDR / stabilization / binned / pixelFormat / resolutionBias constraints | | [visioncamera.controller.harness.ts](visioncamera.controller.harness.ts) | `CameraController` — zoom, torch, exposure bias, focus metering, low-light boost, subject area listener | @@ -228,9 +228,6 @@ file pointing at what needs to land first. Today: `CameraControl.setExposureCompensationIndex` silently fails in that state. The corresponding tests are `it.skip` until the initial config application happens at a point where CameraX accepts it. -- **`enablePreviewSizedOutputBuffers` on Android** — the flag is not honored - by `HybridFrameOutput.kt` today (`TODO: enablePreviewSizedOutputBuffers is - not taken into account here.`). - **`onFrameDropped` on Android** — `HybridFrameOutput.setOnFrameDroppedCallback` is a no-op (`TODO: CameraX does not have a way to figure out if a Frame has been dropped or not.`). diff --git a/apps/simple-camera/__tests__/visioncamera.constraints.harness.ts b/apps/simple-camera/__tests__/visioncamera.constraints.harness.ts index 90addfc8e4..3465a5a85d 100644 --- a/apps/simple-camera/__tests__/visioncamera.constraints.harness.ts +++ b/apps/simple-camera/__tests__/visioncamera.constraints.harness.ts @@ -265,7 +265,7 @@ describe('VisionCamera - Constraints', () => { const frameOutput = VisionCamera.createFrameOutput({ targetResolution: CommonResolutions.HD_16_9, pixelFormat: 'native', - enablePreviewSizedOutputBuffers: false, + allowPhysicalBufferResizing: false, enablePhysicalBufferRotation: false, enableCameraMatrixDelivery: false, allowDeferredStart: false, diff --git a/apps/simple-camera/__tests__/visioncamera.frame.harness.ts b/apps/simple-camera/__tests__/visioncamera.frame.harness.ts index 2cd90cbaa7..5efe540c9a 100644 --- a/apps/simple-camera/__tests__/visioncamera.frame.harness.ts +++ b/apps/simple-camera/__tests__/visioncamera.frame.harness.ts @@ -5,6 +5,7 @@ import { it, waitUntil, } from 'react-native-harness' +import { Platform } from 'react-native' import type { CameraDevice, CameraDeviceFactory, @@ -32,7 +33,7 @@ describe('VisionCamera - Frame', () => { const frameOutput = VisionCamera.createFrameOutput({ targetResolution: CommonResolutions.HD_16_9, pixelFormat: 'native', - enablePreviewSizedOutputBuffers: false, + allowPhysicalBufferResizing: false, enablePhysicalBufferRotation: false, enableCameraMatrixDelivery: false, allowDeferredStart: false, @@ -73,7 +74,7 @@ describe('VisionCamera - Frame', () => { const frameOutput = VisionCamera.createFrameOutput({ targetResolution: CommonResolutions.HD_16_9, pixelFormat: 'yuv', - enablePreviewSizedOutputBuffers: false, + allowPhysicalBufferResizing: false, enablePhysicalBufferRotation: false, enableCameraMatrixDelivery: false, allowDeferredStart: false, @@ -132,7 +133,7 @@ describe('VisionCamera - Frame', () => { const frameOutput = VisionCamera.createFrameOutput({ targetResolution: CommonResolutions.VGA_16_9, pixelFormat: 'rgb', - enablePreviewSizedOutputBuffers: false, + allowPhysicalBufferResizing: false, enablePhysicalBufferRotation: false, enableCameraMatrixDelivery: false, allowDeferredStart: false, @@ -172,7 +173,7 @@ describe('VisionCamera - Frame', () => { const frameOutput = VisionCamera.createFrameOutput({ targetResolution: CommonResolutions.VGA_16_9, pixelFormat: 'native', - enablePreviewSizedOutputBuffers: false, + allowPhysicalBufferResizing: false, enablePhysicalBufferRotation: false, enableCameraMatrixDelivery: false, allowDeferredStart: false, @@ -214,7 +215,7 @@ describe('VisionCamera - Frame', () => { const frameOutput = VisionCamera.createFrameOutput({ targetResolution: CommonResolutions.HD_16_9, pixelFormat: 'native', - enablePreviewSizedOutputBuffers: false, + allowPhysicalBufferResizing: false, enablePhysicalBufferRotation: false, enableCameraMatrixDelivery: false, allowDeferredStart: false, @@ -255,17 +256,25 @@ describe('VisionCamera - Frame', () => { } }) - // TODO: Re-enable once the Android frame output honors `enablePreviewSizedOutputBuffers` - // (today HybridFrameOutput.kt / HybridDepthFrameOutput.kt both have a - // `TODO: enablePreviewSizedOutputBuffers is not taken into account here.`). - // Actually, maybe we should remoev `enablePreviewSizedOutputBuffers` in favor of - // the simple, yet more flexible `targetResolution: ...` prop anyways. - it.skip('delivers smaller buffers when enablePreviewSizedOutputBuffers is true', async () => { + it('downscales frame buffers when allowPhysicalBufferResizing is true (iOS only)', async () => { + if (Platform.OS !== 'ios') { + console.log('[SKIP] allowPhysicalBufferResizing: iOS only') + return + } + // Session hosts both a UHD video output and a VGA-target frame output. + // With `allowPhysicalBufferResizing: true`, the frame output opts out of + // resolution negotiation so the session picks a format suitable for the + // video recorder (≈ UHD), and the frame pipeline physically downscales + // delivered buffers to ≈ VGA via `videoSettings` W/H. const session = await VisionCamera.createCameraSession(false) - const frameOutput = VisionCamera.createFrameOutput({ + const videoOutput = VisionCamera.createVideoOutput({ targetResolution: CommonResolutions.UHD_16_9, + enableAudio: false, + }) + const frameOutput = VisionCamera.createFrameOutput({ + targetResolution: CommonResolutions.VGA_16_9, pixelFormat: 'native', - enablePreviewSizedOutputBuffers: true, + allowPhysicalBufferResizing: true, enablePhysicalBufferRotation: false, enableCameraMatrixDelivery: false, allowDeferredStart: false, @@ -274,7 +283,10 @@ describe('VisionCamera - Frame', () => { await session.configure([ { input: backDevice, - outputs: [{ output: frameOutput, mirrorMode: 'auto' }], + outputs: [ + { output: videoOutput, mirrorMode: 'auto' }, + { output: frameOutput, mirrorMode: 'auto' }, + ], constraints: [], }, ]) @@ -302,12 +314,25 @@ describe('VisionCamera - Frame', () => { runtime.setOnFrameCallback(frameOutput, undefined) await session.stop() } + const frameLong = Math.max(reportedWidth, reportedHeight) + const vgaLong = Math.max( + CommonResolutions.VGA_16_9.width, + CommonResolutions.VGA_16_9.height, + ) + const uhdLong = Math.max( + CommonResolutions.UHD_16_9.width, + CommonResolutions.UHD_16_9.height, + ) console.log( - `preview-sized frame: ${reportedWidth}x${reportedHeight} (requested target ${CommonResolutions.UHD_16_9.width}x${CommonResolutions.UHD_16_9.height})`, + `allowPhysicalBufferResizing frame: ${reportedWidth}x${reportedHeight} (target VGA long ${vgaLong}, session UHD long ${uhdLong})`, ) - const requestedPixels = - CommonResolutions.UHD_16_9.width * CommonResolutions.UHD_16_9.height - const actualPixels = reportedWidth * reportedHeight - expect(actualPixels).toBeLessThan(requestedPixels) + // Downscale happened — frame long side is well below UHD's. + expect(frameLong).toBeLessThan(uhdLong) + // Frame long side lands close to VGA, with tolerance for the + // active-aspect clamp (iOS only accepts sizes matching the active + // format's aspect ratio, so rounding to the nearest legal size is + // expected). Keep the band wide enough to absorb that. + expect(frameLong).toBeGreaterThanOrEqual(480) + expect(frameLong).toBeLessThanOrEqual(1280) }) }) diff --git a/docs/content/docs/a-frame.mdx b/docs/content/docs/a-frame.mdx index 1fc1897cf4..d73d05031c 100644 --- a/docs/content/docs/a-frame.mdx +++ b/docs/content/docs/a-frame.mdx @@ -65,6 +65,32 @@ case .down: > [!TIP] > See ["Orientation"](orientation) for more information about orientation. +### Physical buffer resizing + +By default, a [`CameraFrameOutput`](/api/react-native-vision-camera/hybrid-objects/CameraFrameOutput) participates in the [`CameraSession`](/api/react-native-vision-camera/hybrid-objects/CameraSession)'s resolution negotiation — its [`targetResolution`](/api/react-native-vision-camera/interfaces/FrameOutputOptions#targetresolution) is used as a `"closest to"` bias, and all outputs share the same negotiated format. + +For multi-output sessions where the [`CameraFrameOutput`](/api/react-native-vision-camera/hybrid-objects/CameraFrameOutput) wants smaller buffers than another output needs (e.g. a Frame Processor running ML on preview-sized buffers while a [`CameraVideoOutput`](/api/react-native-vision-camera/hybrid-objects/CameraVideoOutput) concurrently records at 4K), set [`allowPhysicalBufferResizing`](/api/react-native-vision-camera/interfaces/FrameOutputOptions#allowphysicalbufferresizing) to `true`: + +```ts +const frameOutput = useFrameOutput({ + // [!code ++] + targetResolution: { width: 480, height: 854 }, + // [!code ++] + allowPhysicalBufferResizing: true, + onFrame(frame) { + 'worklet' + frame.dispose() + } +}) +``` + +This opts the [`CameraFrameOutput`](/api/react-native-vision-camera/hybrid-objects/CameraFrameOutput) out of session-wide resolution negotiation — the [`CameraSession`](/api/react-native-vision-camera/hybrid-objects/CameraSession) picks the format best for the other outputs — and the Camera pipeline physically downscales delivered [`Frame`](/api/react-native-vision-camera/hybrid-objects/Frame)s to [`targetResolution`](/api/react-native-vision-camera/interfaces/FrameOutputOptions#targetresolution). + +> [!NOTE] +> On iOS, the resized buffer size must match the active format's aspect ratio and cannot exceed its native dimensions. A [`targetResolution`](/api/react-native-vision-camera/interfaces/FrameOutputOptions#targetresolution) whose aspect ratio differs from the active format will be honored along its long side only — the delivered [`Frame`](/api/react-native-vision-camera/hybrid-objects/Frame)'s aspect ratio follows the active format. + +[`allowPhysicalBufferResizing`](/api/react-native-vision-camera/interfaces/FrameOutputOptions#allowphysicalbufferresizing) is iOS-only; Android already uses [`targetResolution`](/api/react-native-vision-camera/interfaces/FrameOutputOptions#targetresolution) as the sole resolution input for [`CameraFrameOutput`](/api/react-native-vision-camera/hybrid-objects/CameraFrameOutput), so the flag has no effect there. + ### Understanding Pixel Formats A [`Frame`](/api/react-native-vision-camera/hybrid-objects/Frame) has a GPU-backed buffer that contains its pixels - their layout is described by the [`Frame`](/api/react-native-vision-camera/hybrid-objects/Frame)'s [`pixelFormat`](/api/react-native-vision-camera/hybrid-objects/Frame#pixelformat). diff --git a/packages/react-native-vision-camera-skia/src/views/SkiaCamera.tsx b/packages/react-native-vision-camera-skia/src/views/SkiaCamera.tsx index d618123c5e..6e69d94656 100644 --- a/packages/react-native-vision-camera-skia/src/views/SkiaCamera.tsx +++ b/packages/react-native-vision-camera-skia/src/views/SkiaCamera.tsx @@ -23,6 +23,7 @@ import { type CameraVideoOutput, type FocusOptions, type Frame, + type FrameOutputOptions, type MeteringMode, type MirrorMode, type PixelFormat, @@ -181,20 +182,23 @@ export interface SkiaCameraProps */ enablePhysicalBufferRotation?: boolean /** - * Configures the underlying {@linkcode CameraVideoOutput} - * to downscale {@linkcode Frame}s to a size suitable for - * preview, such as the screen size. + * Allow the underlying {@linkcode CameraFrameOutput} to physically + * resize its buffers independently of the {@linkcode CameraSession}'s + * negotiated resolution. * - * This may be more efficient if the {@linkcode CameraSession} - * negotiated a high-resolution output, as Skia doesn't - * have to operate on high-resolution {@linkcode Frame}s. - * Ideally, configure a matching {@linkcode ResolutionBiasConstraint} - * upfront so no scaling would be necessary at any point - * in the pipeline. + * This is useful when another output (e.g. a high-resolution + * {@linkcode CameraVideoOutput}) needs a full-resolution format but + * Skia's preview pipeline can render more efficiently from + * downscaled {@linkcode Frame}s. * + * See {@linkcode FrameOutputOptions.allowPhysicalBufferResizing} + * for details and caveats (active-format aspect ratio match, no + * upscaling beyond native). + * + * @platform iOS * @default false */ - enablePreviewSizedOutputBuffers?: boolean + allowPhysicalBufferResizing?: boolean } const DEFAULT_PIXEL_FORMAT = Platform.select({ @@ -209,7 +213,7 @@ function SkiaCameraImpl({ onFrame, pixelFormat = DEFAULT_PIXEL_FORMAT, enablePhysicalBufferRotation, - enablePreviewSizedOutputBuffers, + allowPhysicalBufferResizing, device, orientationSource, warnIfRenderSkipped = true, @@ -252,7 +256,7 @@ function SkiaCameraImpl({ dropFramesWhileBusy: true, allowDeferredStart: false, enablePhysicalBufferRotation: enablePhysicalBufferRotation, - enablePreviewSizedOutputBuffers: enablePreviewSizedOutputBuffers, + allowPhysicalBufferResizing: allowPhysicalBufferResizing, onFrame: (frame) => { 'worklet' let renderCount = 0 diff --git a/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/hybrids/outputs/HybridDepthFrameOutput.kt b/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/hybrids/outputs/HybridDepthFrameOutput.kt index 3cce8a4c14..15a88bed50 100644 --- a/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/hybrids/outputs/HybridDepthFrameOutput.kt +++ b/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/hybrids/outputs/HybridDepthFrameOutput.kt @@ -57,7 +57,6 @@ class HybridDepthFrameOutput( ResolutionSelector .Builder() .setResolutionFilter { sizes, _ -> - // TODO: `enablePreviewSizedOutputBuffers` is not taken into account here. val targetSize = options.targetResolution.toSize() return@setResolutionFilter sizes.sortedByClosestTo(targetSize) }.build() diff --git a/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/hybrids/outputs/HybridFrameOutput.kt b/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/hybrids/outputs/HybridFrameOutput.kt index 560ce298d3..046d8a2108 100644 --- a/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/hybrids/outputs/HybridFrameOutput.kt +++ b/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/hybrids/outputs/HybridFrameOutput.kt @@ -57,7 +57,6 @@ class HybridFrameOutput( ResolutionSelector .Builder() .setResolutionFilter { sizes, _ -> - // TODO: `enablePreviewSizedOutputBuffers` is not taken into account here. val targetSize = options.targetResolution.toSize() return@setResolutionFilter sizes.sortedByClosestTo(targetSize) }.build() diff --git a/packages/react-native-vision-camera/ios/Hybrid Objects/Outputs/HybridCameraFrameOutput.swift b/packages/react-native-vision-camera/ios/Hybrid Objects/Outputs/HybridCameraFrameOutput.swift index d12f43f82d..0fdbb6b323 100644 --- a/packages/react-native-vision-camera/ios/Hybrid Objects/Outputs/HybridCameraFrameOutput.swift +++ b/packages/react-native-vision-camera/ios/Hybrid Objects/Outputs/HybridCameraFrameOutput.swift @@ -33,6 +33,11 @@ class HybridCameraFrameOutput: HybridCameraFrameOutputSpec, NativeCameraOutput { var streamType: StreamType = .video var targetResolution: ResolutionRule { + if options.allowPhysicalBufferResizing { + // Opt out of negotiation — the CameraSession picks whatever format is best for the + // other outputs, and we pin our buffer size via `videoSettings` in `configure(...)`. + return .any + } return .closestTo(options.targetResolution) } @@ -62,14 +67,6 @@ class HybridCameraFrameOutput: HybridCameraFrameOutputSpec, NativeCameraOutput { // of preview-related outputs to make preview appear faster. output.isDeferredStartEnabled = options.allowDeferredStart } - // TODO: Add a flag called `allowBufferResizing` and probably get rid of `enablePreviewSizedOutputBuffers` - // If `allowBufferResizing` is true, we can set `.videoSettings`/dimensions here to `options.targetResolution` - // and `targetResolution` to `.any` (to not participate in resolution negotiations). - // If `allowBufferResizing` is false, we don't set `.videoSettings` ("native" resolution), and reflect our - // target resolution via `targetResolution` to participate in resolution negotiations. - if #available(iOS 17.0, *), options.enablePreviewSizedOutputBuffers { - output.deliversPreviewSizedOutputBuffers = true - } if #available(iOS 26.0, *) { // We don't need HDR metadata, as that's only useful for encoders. output.preservesDynamicHDRMetadata = false @@ -95,6 +92,65 @@ class HybridCameraFrameOutput: HybridCameraFrameOutputSpec, NativeCameraOutput { connection.isCameraIntrinsicMatrixDeliveryEnabled = true } } + if #available(iOS 16.0, *), options.allowPhysicalBufferResizing { + // We opted out of resolution negotiation via `targetResolution = .any`; pin our + // delivered buffer size here via `videoSettings`. iOS 16+ enforces that the + // requested W/H match the active format's aspect ratio (corrected for the + // connection's videoOrientation) and do not exceed its native dimensions. + if let input = connection.inputPorts.first?.input as? AVCaptureDeviceInput, + let dims = HybridCameraFrameOutput.computeResizedBufferDimensions( + activeFormat: input.device.activeFormat, + connection: connection, + target: options.targetResolution) { + var settings = output.videoSettings ?? [:] + settings[kCVPixelBufferWidthKey as String] = dims.width + settings[kCVPixelBufferHeightKey as String] = dims.height + output.videoSettings = settings + } + } + } + + /// Computes a buffer size that satisfies `AVCaptureVideoDataOutput.videoSettings`' + /// iOS 16+ constraints for `kCVPixelBufferWidthKey` / `kCVPixelBufferHeightKey`: + /// • matches the active format's aspect ratio exactly + /// • never exceeds the active format's native dimensions (no upscaling) + /// • is expressed in output (post-rotation) coordinates, respecting + /// the connection's videoOrientation / videoRotationAngle + /// The result is the size closest to `target` that satisfies all of the above. + /// Returns `nil` if the active format has degenerate dimensions. + private static func computeResizedBufferDimensions( + activeFormat: AVCaptureDevice.Format, + connection: AVCaptureConnection, + target: Size + ) -> (width: Int, height: Int)? { + let dims = CMVideoFormatDescriptionGetDimensions(activeFormat.formatDescription) + let activeLong = Int(max(dims.width, dims.height)) + let activeShort = Int(min(dims.width, dims.height)) + guard activeLong > 0, activeShort > 0 else { return nil } + + // Work in orientation-independent (long, short) space. The active format's aspect + // is the only aspect iOS will accept, so we ignore the target's own aspect and + // match the target's long side to the active's. + let targetLong = max(Int(target.width), Int(target.height)) + let useLong = max(min(targetLong, activeLong), 2) + let useShort = max( + Int((Double(useLong) * Double(activeShort) / Double(activeLong)).rounded()), 2) + + // Project back onto (width, height) using the connection's orientation. + let isPortrait: Bool + if #available(iOS 17.0, *) { + let angle = connection.videoRotationAngle + isPortrait = abs(angle - 90).isLess(than: 0.5) || abs(angle - 270).isLess(than: 0.5) + } else { + isPortrait = + connection.videoOrientation == .portrait + || connection.videoOrientation == .portraitUpsideDown + } + if isPortrait { + return (width: useShort, height: useLong) + } else { + return (width: useLong, height: useShort) + } } private func videoSettingsForPixelFormat(_ targetPixelFormat: TargetVideoPixelFormat) -> [String: diff --git a/packages/react-native-vision-camera/src/hooks/useFrameOutput.ts b/packages/react-native-vision-camera/src/hooks/useFrameOutput.ts index 18e7f6be93..c7179fd878 100644 --- a/packages/react-native-vision-camera/src/hooks/useFrameOutput.ts +++ b/packages/react-native-vision-camera/src/hooks/useFrameOutput.ts @@ -124,7 +124,7 @@ export function useFrameOutput({ dropFramesWhileBusy = true, enableCameraMatrixDelivery = false, enablePhysicalBufferRotation = false, - enablePreviewSizedOutputBuffers = false, + allowPhysicalBufferResizing = false, allowDeferredStart = true, onFrame, onFrameDropped, @@ -137,7 +137,7 @@ export function useFrameOutput({ pixelFormat: pixelFormat, enablePhysicalBufferRotation: enablePhysicalBufferRotation, enableCameraMatrixDelivery: enableCameraMatrixDelivery, - enablePreviewSizedOutputBuffers: enablePreviewSizedOutputBuffers, + allowPhysicalBufferResizing: allowPhysicalBufferResizing, allowDeferredStart: allowDeferredStart, dropFramesWhileBusy: dropFramesWhileBusy, }), @@ -147,7 +147,7 @@ export function useFrameOutput({ dropFramesWhileBusy, enableCameraMatrixDelivery, enablePhysicalBufferRotation, - enablePreviewSizedOutputBuffers, + allowPhysicalBufferResizing, allowDeferredStart, ], ) diff --git a/packages/react-native-vision-camera/src/specs/outputs/CameraFrameOutput.nitro.ts b/packages/react-native-vision-camera/src/specs/outputs/CameraFrameOutput.nitro.ts index 70c5de04c5..0af919decb 100644 --- a/packages/react-native-vision-camera/src/specs/outputs/CameraFrameOutput.nitro.ts +++ b/packages/react-native-vision-camera/src/specs/outputs/CameraFrameOutput.nitro.ts @@ -10,8 +10,6 @@ import type { CameraSession } from '../session/CameraSession.nitro' import type { CameraSessionConfig } from '../session/CameraSessionConfig.nitro' import type { CameraOutput } from './CameraOutput.nitro' -// TODO: Should we remove `enablePreviewSizedOutputBuffers` and infer it from `targetResolution`? - /** * Configuration options for a {@linkcode CameraFrameOutput}. * @@ -38,18 +36,39 @@ export interface FrameOutputOptions { targetResolution: Size /** - * Deliver smaller, preview-sized output buffers for Frame Processing. - * - * This is useful for ML and computer vision workloads where full-resolution - * buffers are unnecessary and would only increase memory bandwidth and - * processing costs. + * Allow the {@linkcode CameraFrameOutput} to physically resize its + * buffers independently of the {@linkcode CameraSession}'s negotiated + * resolution. + * + * - When `false` (default), the {@linkcode CameraFrameOutput} participates + * in the {@linkcode CameraSession}'s resolution negotiation using its + * {@linkcode targetResolution} as a bias (see "closest to"). All outputs + * share the same negotiated format, so the {@linkcode CameraFrameOutput} + * receives {@linkcode Frame}s at the negotiated resolution regardless of + * what other outputs (e.g. a full-resolution {@linkcode CameraVideoOutput}) + * requested. + * - When `true`, the {@linkcode CameraFrameOutput} opts _out_ of + * resolution negotiation — the {@linkcode CameraSession} is free to + * pick the best format for the _other_ outputs — and the Camera pipeline + * physically downscales delivered {@linkcode Frame}s to match + * {@linkcode targetResolution} as closely as possible. This is useful when + * the {@linkcode CameraFrameOutput} wants small buffers (e.g. for ML or + * GPU preview) while another output (e.g. a 4K {@linkcode CameraVideoOutput}) + * needs to keep using the full-resolution output. * - * Other camera outputs (for example {@linkcode CameraVideoOutput}) keep using - * the full-resolution output negotiated by the {@linkcode CameraSession}. + * @discussion + * The "closely as possible" caveat is important: the underlying + * `AVCaptureVideoDataOutput` will only accept a physically-resized buffer size + * that matches the active format's aspect ratio and does not exceed its + * native dimensions. A {@linkcode targetResolution} that does not match + * the active format's aspect ratio will therefore be honored along its + * long side only — the aspect ratio of the delivered {@linkcode Frame} + * follows the active format. * + * @platform iOS * @default false */ - enablePreviewSizedOutputBuffers: boolean + allowPhysicalBufferResizing: boolean /** * Allow this output to start later in the capture pipeline startup process.