feat: Add allowPhysicalBufferResizing, remove enablePreviewSizedOutputBuffers

apps/simple-camera/__tests__/README.md

@@ -39,7 +39,7 @@ Tests are split by domain. Each file tests one slice of the imperative
 | [visioncamera.session.harness.ts](visioncamera.session.harness.ts) | `createCameraSession`, `configure`, `start`, `stop`, `addOnStartedListener` / `addOnStoppedListener` / `addOnErrorListener` / interruption listeners, reconfigure-while-running, multi-cam |
 | [visioncamera.photo.harness.ts](visioncamera.photo.harness.ts) | `createPhotoOutput`, `capturePhoto` / `capturePhotoToFile`, container formats (JPEG, HEIC, DNG), flash / mirror / quality / resolution options, capture lifecycle callbacks, preview images |
 | [visioncamera.video.harness.ts](visioncamera.video.harness.ts) | `createVideoOutput`, `Recorder` lifecycle, audio, `maxDuration` / `maxFileSize` stops, pause / resume / cancel, persistent recorder, higher-resolution codecs |
-| [visioncamera.frame.harness.ts](visioncamera.frame.harness.ts) | `createFrameOutput`, worklet install via `react-native-vision-camera-worklets`, YUV / RGB / native pixel formats, `scheduleOnRN`, `createSynchronizable`, `setOnFrameDroppedCallback`, `enablePreviewSizedOutputBuffers` |
+| [visioncamera.frame.harness.ts](visioncamera.frame.harness.ts) | `createFrameOutput`, worklet install via `react-native-vision-camera-worklets`, YUV / RGB / native pixel formats, `scheduleOnRN`, `createSynchronizable`, `setOnFrameDroppedCallback`, `allowPhysicalBufferResizing` |
 | [visioncamera.constraints.harness.ts](visioncamera.constraints.harness.ts) | `VisionCamera.resolveConstraints` + `onSessionConfigSelected`, FPS / HDR / stabilization / binned / pixelFormat / resolutionBias constraints |
 | [visioncamera.controller.harness.ts](visioncamera.controller.harness.ts) | `CameraController` — zoom, torch, exposure bias, focus metering, low-light boost, subject area listener |
 
@@ -228,9 +228,6 @@ file pointing at what needs to land first. Today:
   `CameraControl.setExposureCompensationIndex` silently fails in that state.
   The corresponding tests are `it.skip` until the initial config application
   happens at a point where CameraX accepts it.
-- **`enablePreviewSizedOutputBuffers` on Android** — the flag is not honored
-  by `HybridFrameOutput.kt` today (`TODO: enablePreviewSizedOutputBuffers is
-  not taken into account here.`).
 - **`onFrameDropped` on Android** — `HybridFrameOutput.setOnFrameDroppedCallback`
   is a no-op (`TODO: CameraX does not have a way to figure out if a Frame
   has been dropped or not.`).

apps/simple-camera/__tests__/visioncamera.constraints.harness.ts

@@ -265,7 +265,7 @@ describe('VisionCamera - Constraints', () => {
     const frameOutput = VisionCamera.createFrameOutput({
       targetResolution: CommonResolutions.HD_16_9,
       pixelFormat: 'native',
-      enablePreviewSizedOutputBuffers: false,
+      allowPhysicalBufferResizing: false,
       enablePhysicalBufferRotation: false,
       enableCameraMatrixDelivery: false,
       allowDeferredStart: false,

apps/simple-camera/__tests__/visioncamera.frame.harness.ts

@@ -5,6 +5,7 @@ import {
   it,
   waitUntil,
 } from 'react-native-harness'
+import { Platform } from 'react-native'
 import type {
   CameraDevice,
   CameraDeviceFactory,
@@ -32,7 +33,7 @@ describe('VisionCamera - Frame', () => {
     const frameOutput = VisionCamera.createFrameOutput({
       targetResolution: CommonResolutions.HD_16_9,
       pixelFormat: 'native',
-      enablePreviewSizedOutputBuffers: false,
+      allowPhysicalBufferResizing: false,
       enablePhysicalBufferRotation: false,
       enableCameraMatrixDelivery: false,
       allowDeferredStart: false,
@@ -73,7 +74,7 @@ describe('VisionCamera - Frame', () => {
     const frameOutput = VisionCamera.createFrameOutput({
       targetResolution: CommonResolutions.HD_16_9,
       pixelFormat: 'yuv',
-      enablePreviewSizedOutputBuffers: false,
+      allowPhysicalBufferResizing: false,
       enablePhysicalBufferRotation: false,
       enableCameraMatrixDelivery: false,
       allowDeferredStart: false,
@@ -132,7 +133,7 @@ describe('VisionCamera - Frame', () => {
     const frameOutput = VisionCamera.createFrameOutput({
       targetResolution: CommonResolutions.VGA_16_9,
       pixelFormat: 'rgb',
-      enablePreviewSizedOutputBuffers: false,
+      allowPhysicalBufferResizing: false,
       enablePhysicalBufferRotation: false,
       enableCameraMatrixDelivery: false,
       allowDeferredStart: false,
@@ -172,7 +173,7 @@ describe('VisionCamera - Frame', () => {
     const frameOutput = VisionCamera.createFrameOutput({
       targetResolution: CommonResolutions.VGA_16_9,
       pixelFormat: 'native',
-      enablePreviewSizedOutputBuffers: false,
+      allowPhysicalBufferResizing: false,
       enablePhysicalBufferRotation: false,
       enableCameraMatrixDelivery: false,
       allowDeferredStart: false,
@@ -214,7 +215,7 @@ describe('VisionCamera - Frame', () => {
     const frameOutput = VisionCamera.createFrameOutput({
       targetResolution: CommonResolutions.HD_16_9,
       pixelFormat: 'native',
-      enablePreviewSizedOutputBuffers: false,
+      allowPhysicalBufferResizing: false,
       enablePhysicalBufferRotation: false,
       enableCameraMatrixDelivery: false,
       allowDeferredStart: false,
@@ -255,17 +256,25 @@ describe('VisionCamera - Frame', () => {
     }
   })
 
-  // TODO: Re-enable once the Android frame output honors `enablePreviewSizedOutputBuffers`
-  //       (today HybridFrameOutput.kt / HybridDepthFrameOutput.kt both have a
-  //       `TODO: enablePreviewSizedOutputBuffers is not taken into account here.`).
-  //       Actually, maybe we should remoev `enablePreviewSizedOutputBuffers` in favor of
-  //       the simple, yet more flexible `targetResolution: ...` prop anyways.
-  it.skip('delivers smaller buffers when enablePreviewSizedOutputBuffers is true', async () => {
+  it('downscales frame buffers when allowPhysicalBufferResizing is true (iOS only)', async () => {
+    if (Platform.OS !== 'ios') {
+      console.log('[SKIP] allowPhysicalBufferResizing: iOS only')
+      return
+    }
+    // Session hosts both a UHD video output and a VGA-target frame output.
+    // With `allowPhysicalBufferResizing: true`, the frame output opts out of
+    // resolution negotiation so the session picks a format suitable for the
+    // video recorder (≈ UHD), and the frame pipeline physically downscales
+    // delivered buffers to ≈ VGA via `videoSettings` W/H.
     const session = await VisionCamera.createCameraSession(false)
-    const frameOutput = VisionCamera.createFrameOutput({
+    const videoOutput = VisionCamera.createVideoOutput({
       targetResolution: CommonResolutions.UHD_16_9,
+      enableAudio: false,
+    })
+    const frameOutput = VisionCamera.createFrameOutput({
+      targetResolution: CommonResolutions.VGA_16_9,
       pixelFormat: 'native',
-      enablePreviewSizedOutputBuffers: true,
+      allowPhysicalBufferResizing: true,
       enablePhysicalBufferRotation: false,
       enableCameraMatrixDelivery: false,
       allowDeferredStart: false,
@@ -274,7 +283,10 @@ describe('VisionCamera - Frame', () => {
     await session.configure([
       {
         input: backDevice,
-        outputs: [{ output: frameOutput, mirrorMode: 'auto' }],
+        outputs: [
+          { output: videoOutput, mirrorMode: 'auto' },
+          { output: frameOutput, mirrorMode: 'auto' },
+        ],
         constraints: [],
       },
     ])
@@ -302,12 +314,25 @@ describe('VisionCamera - Frame', () => {
       runtime.setOnFrameCallback(frameOutput, undefined)
       await session.stop()
     }
+    const frameLong = Math.max(reportedWidth, reportedHeight)
+    const vgaLong = Math.max(
+      CommonResolutions.VGA_16_9.width,
+      CommonResolutions.VGA_16_9.height,
+    )
+    const uhdLong = Math.max(
+      CommonResolutions.UHD_16_9.width,
+      CommonResolutions.UHD_16_9.height,
+    )
     console.log(
-      `preview-sized frame: ${reportedWidth}x${reportedHeight} (requested target ${CommonResolutions.UHD_16_9.width}x${CommonResolutions.UHD_16_9.height})`,
+      `allowPhysicalBufferResizing frame: ${reportedWidth}x${reportedHeight} (target VGA long ${vgaLong}, session UHD long ${uhdLong})`,
     )
-    const requestedPixels =
-      CommonResolutions.UHD_16_9.width * CommonResolutions.UHD_16_9.height
-    const actualPixels = reportedWidth * reportedHeight
-    expect(actualPixels).toBeLessThan(requestedPixels)
+    // Downscale happened — frame long side is well below UHD's.
+    expect(frameLong).toBeLessThan(uhdLong)
+    // Frame long side lands close to VGA, with tolerance for the
+    // active-aspect clamp (iOS only accepts sizes matching the active
+    // format's aspect ratio, so rounding to the nearest legal size is
+    // expected). Keep the band wide enough to absorb that.
+    expect(frameLong).toBeGreaterThanOrEqual(480)
+    expect(frameLong).toBeLessThanOrEqual(1280)
   })
 })

docs/content/docs/a-frame.mdx

@@ -65,6 +65,32 @@ case .down:
 > [!TIP]
 > See ["Orientation"](orientation) for more information about orientation.
 
+### Physical buffer resizing
+
+By default, a [`CameraFrameOutput`](/api/react-native-vision-camera/hybrid-objects/CameraFrameOutput) participates in the [`CameraSession`](/api/react-native-vision-camera/hybrid-objects/CameraSession)'s resolution negotiation — its [`targetResolution`](/api/react-native-vision-camera/interfaces/FrameOutputOptions#targetresolution) is used as a `"closest to"` bias, and all outputs share the same negotiated format.
+
+For multi-output sessions where the [`CameraFrameOutput`](/api/react-native-vision-camera/hybrid-objects/CameraFrameOutput) wants smaller buffers than another output needs (e.g. a Frame Processor running ML on preview-sized buffers while a [`CameraVideoOutput`](/api/react-native-vision-camera/hybrid-objects/CameraVideoOutput) concurrently records at 4K), set [`allowPhysicalBufferResizing`](/api/react-native-vision-camera/interfaces/FrameOutputOptions#allowphysicalbufferresizing) to `true`:
+
+```ts
+const frameOutput = useFrameOutput({
+  // [!code ++]
+  targetResolution: { width: 480, height: 854 },
+  // [!code ++]
+  allowPhysicalBufferResizing: true,
+  onFrame(frame) {
+    'worklet'
+    frame.dispose()
+  }
+})
+```
+
+This opts the [`CameraFrameOutput`](/api/react-native-vision-camera/hybrid-objects/CameraFrameOutput) out of session-wide resolution negotiation — the [`CameraSession`](/api/react-native-vision-camera/hybrid-objects/CameraSession) picks the format best for the other outputs — and the Camera pipeline physically downscales delivered [`Frame`](/api/react-native-vision-camera/hybrid-objects/Frame)s to [`targetResolution`](/api/react-native-vision-camera/interfaces/FrameOutputOptions#targetresolution).
+
+> [!NOTE]
+> On iOS, the resized buffer size must match the active format's aspect ratio and cannot exceed its native dimensions. A [`targetResolution`](/api/react-native-vision-camera/interfaces/FrameOutputOptions#targetresolution) whose aspect ratio differs from the active format will be honored along its long side only — the delivered [`Frame`](/api/react-native-vision-camera/hybrid-objects/Frame)'s aspect ratio follows the active format.
+
+[`allowPhysicalBufferResizing`](/api/react-native-vision-camera/interfaces/FrameOutputOptions#allowphysicalbufferresizing) is iOS-only; Android already uses [`targetResolution`](/api/react-native-vision-camera/interfaces/FrameOutputOptions#targetresolution) as the sole resolution input for [`CameraFrameOutput`](/api/react-native-vision-camera/hybrid-objects/CameraFrameOutput), so the flag has no effect there.
+
 ### Understanding Pixel Formats
 
 A [`Frame`](/api/react-native-vision-camera/hybrid-objects/Frame) has a GPU-backed buffer that contains its pixels - their layout is described by the [`Frame`](/api/react-native-vision-camera/hybrid-objects/Frame)'s [`pixelFormat`](/api/react-native-vision-camera/hybrid-objects/Frame#pixelformat).

packages/react-native-vision-camera-skia/src/views/SkiaCamera.tsx

@@ -23,6 +23,7 @@ import {
   type CameraVideoOutput,
   type FocusOptions,
   type Frame,
+  type FrameOutputOptions,
   type MeteringMode,
   type MirrorMode,
   type PixelFormat,
@@ -181,20 +182,23 @@ export interface SkiaCameraProps
    */
   enablePhysicalBufferRotation?: boolean
   /**
-   * Configures the underlying {@linkcode CameraVideoOutput}
-   * to downscale {@linkcode Frame}s to a size suitable for
-   * preview, such as the screen size.
+   * Allow the underlying {@linkcode CameraFrameOutput} to physically
+   * resize its buffers independently of the {@linkcode CameraSession}'s
+   * negotiated resolution.
    *
-   * This may be more efficient if the {@linkcode CameraSession}
-   * negotiated a high-resolution output, as Skia doesn't
-   * have to operate on high-resolution {@linkcode Frame}s.
-   * Ideally, configure a matching {@linkcode ResolutionBiasConstraint}
-   * upfront so no scaling would be necessary at any point
-   * in the pipeline.
+   * This is useful when another output (e.g. a high-resolution
+   * {@linkcode CameraVideoOutput}) needs a full-resolution format but
+   * Skia's preview pipeline can render more efficiently from
+   * downscaled {@linkcode Frame}s.
    *
+   * See {@linkcode FrameOutputOptions.allowPhysicalBufferResizing}
+   * for details and caveats (active-format aspect ratio match, no
+   * upscaling beyond native).
+   *
+   * @platform iOS
    * @default false
    */
-  enablePreviewSizedOutputBuffers?: boolean
+  allowPhysicalBufferResizing?: boolean
 }
 
 const DEFAULT_PIXEL_FORMAT = Platform.select<TargetVideoPixelFormat>({
@@ -209,7 +213,7 @@ function SkiaCameraImpl({
   onFrame,
   pixelFormat = DEFAULT_PIXEL_FORMAT,
   enablePhysicalBufferRotation,
-  enablePreviewSizedOutputBuffers,
+  allowPhysicalBufferResizing,
   device,
   orientationSource,
   warnIfRenderSkipped = true,
@@ -252,7 +256,7 @@ function SkiaCameraImpl({
     dropFramesWhileBusy: true,
     allowDeferredStart: false,
     enablePhysicalBufferRotation: enablePhysicalBufferRotation,
-    enablePreviewSizedOutputBuffers: enablePreviewSizedOutputBuffers,
+    allowPhysicalBufferResizing: allowPhysicalBufferResizing,
     onFrame: (frame) => {
       'worklet'
       let renderCount = 0

packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/hybrids/outputs/HybridDepthFrameOutput.kt

@@ -57,7 +57,6 @@ class HybridDepthFrameOutput(
       ResolutionSelector
         .Builder()
         .setResolutionFilter { sizes, _ ->
-          // TODO: `enablePreviewSizedOutputBuffers` is not taken into account here.
           val targetSize = options.targetResolution.toSize()
           return@setResolutionFilter sizes.sortedByClosestTo(targetSize)
         }.build()

packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/hybrids/outputs/HybridFrameOutput.kt

@@ -57,7 +57,6 @@ class HybridFrameOutput(
       ResolutionSelector
         .Builder()
         .setResolutionFilter { sizes, _ ->
-          // TODO: `enablePreviewSizedOutputBuffers` is not taken into account here.
           val targetSize = options.targetResolution.toSize()
           return@setResolutionFilter sizes.sortedByClosestTo(targetSize)
         }.build()

packages/react-native-vision-camera/ios/Hybrid Objects/Outputs/HybridCameraFrameOutput.swift

@@ -33,6 +33,11 @@
 
   var streamType: StreamType = .video
   var targetResolution: ResolutionRule {
+    if options.allowPhysicalBufferResizing {
+      // Opt out of negotiation — the CameraSession picks whatever format is best for the
+      // other outputs, and we pin our buffer size via `videoSettings` in `configure(...)`.
+      return .any
+    }
     return .closestTo(options.targetResolution)
   }
 
@@ -62,14 +67,6 @@
       // of preview-related outputs to make preview appear faster.
       output.isDeferredStartEnabled = options.allowDeferredStart
     }
-    // TODO: Add a flag called `allowBufferResizing` and probably get rid of `enablePreviewSizedOutputBuffers`
-    //       If `allowBufferResizing` is true, we can set `.videoSettings`/dimensions here to `options.targetResolution`
-    //       and `targetResolution` to `.any` (to not participate in resolution negotiations).
-    //       If `allowBufferResizing` is false, we don't set `.videoSettings` ("native" resolution), and reflect our
-    //       target resolution via `targetResolution` to participate in resolution negotiations.
-    if #available(iOS 17.0, *), options.enablePreviewSizedOutputBuffers {
-      output.deliversPreviewSizedOutputBuffers = true
-    }
     if #available(iOS 26.0, *) {
       // We don't need HDR metadata, as that's only useful for encoders.
       output.preservesDynamicHDRMetadata = false
@@ -95,6 +92,65 @@
         connection.isCameraIntrinsicMatrixDeliveryEnabled = true
       }
     }
+    if #available(iOS 16.0, *), options.allowPhysicalBufferResizing {
+      // We opted out of resolution negotiation via `targetResolution = .any`; pin our
+      // delivered buffer size here via `videoSettings`. iOS 16+ enforces that the
+      // requested W/H match the active format's aspect ratio (corrected for the
+      // connection's videoOrientation) and do not exceed its native dimensions.
+      if let input = connection.inputPorts.first?.input as? AVCaptureDeviceInput,
+        let dims = HybridCameraFrameOutput.computeResizedBufferDimensions(
+          activeFormat: input.device.activeFormat,
+          connection: connection,
+          target: options.targetResolution) {
+        var settings = output.videoSettings ?? [:]
+        settings[kCVPixelBufferWidthKey as String] = dims.width
+        settings[kCVPixelBufferHeightKey as String] = dims.height
+        output.videoSettings = settings
+      }
+    }
+  }
+
+  /// Computes a buffer size that satisfies `AVCaptureVideoDataOutput.videoSettings`'
+  /// iOS 16+ constraints for `kCVPixelBufferWidthKey` / `kCVPixelBufferHeightKey`:
+  ///   • matches the active format's aspect ratio exactly
+  ///   • never exceeds the active format's native dimensions (no upscaling)
+  ///   • is expressed in output (post-rotation) coordinates, respecting
+  ///     the connection's videoOrientation / videoRotationAngle
+  /// The result is the size closest to `target` that satisfies all of the above.
+  /// Returns `nil` if the active format has degenerate dimensions.
+  private static func computeResizedBufferDimensions(
+    activeFormat: AVCaptureDevice.Format,
+    connection: AVCaptureConnection,
+    target: Size
+  ) -> (width: Int, height: Int)? {
+    let dims = CMVideoFormatDescriptionGetDimensions(activeFormat.formatDescription)
+    let activeLong = Int(max(dims.width, dims.height))
+    let activeShort = Int(min(dims.width, dims.height))
+    guard activeLong > 0, activeShort > 0 else { return nil }
+
+    // Work in orientation-independent (long, short) space. The active format's aspect
+    // is the only aspect iOS will accept, so we ignore the target's own aspect and
+    // match the target's long side to the active's.
+    let targetLong = max(Int(target.width), Int(target.height))
+    let useLong = max(min(targetLong, activeLong), 2)
+    let useShort = max(
+      Int((Double(useLong) * Double(activeShort) / Double(activeLong)).rounded()), 2)
+
+    // Project back onto (width, height) using the connection's orientation.
+    let isPortrait: Bool
+    if #available(iOS 17.0, *) {
+      let angle = connection.videoRotationAngle
+      isPortrait = abs(angle - 90).isLess(than: 0.5) || abs(angle - 270).isLess(than: 0.5)
+    } else {
+      isPortrait =
+        connection.videoOrientation == .portrait
+        || connection.videoOrientation == .portraitUpsideDown
+    }
+    if isPortrait {
+      return (width: useShort, height: useLong)
+    } else {
+      return (width: useLong, height: useShort)
+    }
   }
 
   private func videoSettingsForPixelFormat(_ targetPixelFormat: TargetVideoPixelFormat) -> [String: