fix: Fix coordinate system conversions

apps/simple-camera/__tests__/visioncamera.coordinates.harness.tsx

@@ -490,7 +490,15 @@ describe('VisionCamera - Coordinates', () => {
     const runtime = workletsProvider.createRuntimeForThread(frameOutput.thread)
     runtime.setOnFrameCallback(frameOutput, (frame) => {
       'worklet'
-      const center = { x: frame.width / 2, y: frame.height / 2 }
+      const w = frame.width
+      const h = frame.height
+      const isRotated =
+        frame.orientation === 'left' || frame.orientation === 'right'
+      // Use oriented-space center: for rotated frames the width/height axes swap
+      const center = {
+        x: isRotated ? h / 2 : w / 2,
+        y: isRotated ? w / 2 : h / 2,
+      }
       const cameraPoint = frame.convertFramePointToCameraPoint(center)
       scheduleOnRN(onSample, cameraPoint, frame.orientation)
       frame.dispose()
@@ -530,10 +538,10 @@ describe('VisionCamera - Coordinates', () => {
     }
   })
 
-  // Analyzer coordinates may be reported in the Frame's intended/oriented
-  // image space, while Frame.convertFramePointToCameraPoint consumes raw
-  // buffer-space points. The center-only test above cannot catch an
-  // off-center rectangle drifting after orientation is applied.
+  // Analyzer coordinates are in the Frame's intended/oriented image space.
+  // Frame.convertFramePointToCameraPoint must account for the frame
+  // orientation; the center-only test above cannot catch an off-center
+  // rectangle drifting after orientation is applied.
   // See https://github.com/mrousavy/react-native-vision-camera/pull/3878.
   it('maps oriented Frame rectangles into the same Camera bounds', async () => {
     const session = await VisionCamera.createCameraSession(false)
@@ -554,16 +562,10 @@ describe('VisionCamera - Coordinates', () => {
       },
     ])
 
-    type Bounds = {
-      left: number
-      top: number
-      right: number
-      bottom: number
-    }
     type ProjectionReport = {
       orientation: string
-      expected: Bounds
-      reported: Bounds
+      orientedCorners: Point[]
+      recoveredCorners: Point[]
     }
     let report: ProjectionReport | undefined
     const onReport = (r: ProjectionReport) => {
@@ -598,38 +600,20 @@ describe('VisionCamera - Coordinates', () => {
         { x: box.left, y: box.bottom },
       ]
 
-      const orientedPointToFramePoint = (point: Point): Point => {
-        switch (frame.orientation) {
-          case 'right':
-            return { x: w - point.y, y: point.x }
-          case 'left':
-            return { x: point.y, y: h - point.x }
-          case 'down':
-            return { x: w - point.x, y: h - point.y }
-          default:
-            return point
-        }
-      }
-      const getCameraBounds = (points: Point[]): Bounds => {
-        const cameraPoints = points.map((point) =>
-          frame.convertFramePointToCameraPoint(point),
-        )
-        const xs = cameraPoints.map((point) => point.x)
-        const ys = cameraPoints.map((point) => point.y)
-        return {
-          left: Math.min(...xs),
-          top: Math.min(...ys),
-          right: Math.max(...xs),
-          bottom: Math.max(...ys),
-        }
-      }
+      // Verify round-trip: oriented → camera → oriented should be identity.
+      // This catches the bug where orientation is ignored and the conversion
+      // drifts for off-center points.
+      const cameraCorners = orientedCorners.map((c) =>
+        frame.convertFramePointToCameraPoint(c),
+      )
+      const recoveredCorners = cameraCorners.map((c) =>
+        frame.convertCameraPointToFramePoint(c),
+      )
 
       scheduleOnRN(onReport, {
         orientation: frame.orientation,
-        expected: getCameraBounds(
-          orientedCorners.map(orientedPointToFramePoint),
-        ),
-        reported: getCameraBounds(orientedCorners),
+        orientedCorners: orientedCorners,
+        recoveredCorners: recoveredCorners,
       })
       frame.dispose()
     })
@@ -650,12 +634,18 @@ describe('VisionCamera - Coordinates', () => {
         return
       }
 
-      for (const edge of ['left', 'top', 'right', 'bottom'] as const) {
-        expect(r.reported[edge]).toBeCloseTo(r.expected[edge], 0)
+      // Each oriented corner, after converting to camera and back, must
+      // round-trip to within 1 pixel.  If orientation is ignored the
+      // off-center points drift by (roughly) half a frame dimension.
+      for (let i = 0; i < r.orientedCorners.length; i++) {
+        const original = r.orientedCorners[i]
+        const recovered = r.recoveredCorners[i]
+        expect(recovered.x).toBeCloseTo(original.x, 0)
+        expect(recovered.y).toBeCloseTo(original.y, 0)
       }
 
       console.log(
-        `oriented rectangle projection orientation=${r.orientation} expected=${JSON.stringify(r.expected)} reported=${JSON.stringify(r.reported)}`,
+        `oriented rectangle projection orientation=${r.orientation} corners=${JSON.stringify(r.orientedCorners)} recovered=${JSON.stringify(r.recoveredCorners)}`,
       )
     } finally {
       runtime.setOnFrameCallback(frameOutput, undefined)

packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/hybrids/instances/HybridDepthFrame.kt

@@ -112,15 +112,33 @@ class HybridDepthFrame(
 
   override fun convertCameraPointToDepthPoint(cameraPoint: Point): Point {
     val sensorToBuffer = image.imageInfo.sensorToBufferTransformMatrix
-    return sensorToBuffer.convertPoint(cameraPoint)
+    val rawFramePoint = sensorToBuffer.convertPoint(cameraPoint)
+    // Convert from raw buffer space to oriented (display) space
+    val w = image.width.toDouble()
+    val h = image.height.toDouble()
+    return when (orientation) {
+      CameraOrientation.RIGHT -> Point(rawFramePoint.y, w - rawFramePoint.x)
+      CameraOrientation.LEFT -> Point(h - rawFramePoint.y, rawFramePoint.x)
+      CameraOrientation.DOWN -> Point(w - rawFramePoint.x, h - rawFramePoint.y)
+      CameraOrientation.UP -> rawFramePoint
+    }
   }
 
   override fun convertDepthPointToCameraPoint(depthPoint: Point): Point {
+    // Convert from oriented (display) space to raw buffer space
+    val w = image.width.toDouble()
+    val h = image.height.toDouble()
+    val rawFramePoint = when (orientation) {
+      CameraOrientation.RIGHT -> Point(w - depthPoint.y, depthPoint.x)
+      CameraOrientation.LEFT -> Point(depthPoint.y, h - depthPoint.x)
+      CameraOrientation.DOWN -> Point(w - depthPoint.x, h - depthPoint.y)
+      CameraOrientation.UP -> depthPoint
+    }
     val bufferToSensor =
       Matrix().apply {
         image.imageInfo.sensorToBufferTransformMatrix.invert(this)
       }
-    return bufferToSensor.convertPoint(depthPoint)
+    return bufferToSensor.convertPoint(rawFramePoint)
   }
 
   override fun toFrame(): HybridFrameSpec {

packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/hybrids/instances/HybridFrame.kt

@@ -95,14 +95,32 @@ class HybridFrame(
 
   override fun convertCameraPointToFramePoint(cameraPoint: Point): Point {
     val sensorToBuffer = image.imageInfo.sensorToBufferTransformMatrix
-    return sensorToBuffer.convertPoint(cameraPoint)
+    val rawFramePoint = sensorToBuffer.convertPoint(cameraPoint)
+    // Convert from raw buffer space to oriented (display) space
+    val w = image.width.toDouble()
+    val h = image.height.toDouble()
+    return when (orientation) {
+      CameraOrientation.RIGHT -> Point(rawFramePoint.y, w - rawFramePoint.x)
+      CameraOrientation.LEFT -> Point(h - rawFramePoint.y, rawFramePoint.x)
+      CameraOrientation.DOWN -> Point(w - rawFramePoint.x, h - rawFramePoint.y)
+      CameraOrientation.UP -> rawFramePoint
+    }
   }
 
   override fun convertFramePointToCameraPoint(framePoint: Point): Point {
+    // Convert from oriented (display) space to raw buffer space
+    val w = image.width.toDouble()
+    val h = image.height.toDouble()
+    val rawFramePoint = when (orientation) {
+      CameraOrientation.RIGHT -> Point(w - framePoint.y, framePoint.x)
+      CameraOrientation.LEFT -> Point(framePoint.y, h - framePoint.x)
+      CameraOrientation.DOWN -> Point(w - framePoint.x, h - framePoint.y)
+      CameraOrientation.UP -> framePoint
+    }
     val bufferToSensor =
       Matrix().apply {
         image.imageInfo.sensorToBufferTransformMatrix.invert(this)
       }
-    return bufferToSensor.convertPoint(framePoint)
+    return bufferToSensor.convertPoint(rawFramePoint)
   }
 }