diff --git a/apps/simple-camera/__tests__/visioncamera.coordinates.harness.tsx b/apps/simple-camera/__tests__/visioncamera.coordinates.harness.tsx index af2c7a2475..d14c7e1b99 100644 --- a/apps/simple-camera/__tests__/visioncamera.coordinates.harness.tsx +++ b/apps/simple-camera/__tests__/visioncamera.coordinates.harness.tsx @@ -490,7 +490,15 @@ describe('VisionCamera - Coordinates', () => { const runtime = workletsProvider.createRuntimeForThread(frameOutput.thread) runtime.setOnFrameCallback(frameOutput, (frame) => { 'worklet' - const center = { x: frame.width / 2, y: frame.height / 2 } + const w = frame.width + const h = frame.height + const isRotated = + frame.orientation === 'left' || frame.orientation === 'right' + // Use oriented-space center: for rotated frames the width/height axes swap + const center = { + x: isRotated ? h / 2 : w / 2, + y: isRotated ? w / 2 : h / 2, + } const cameraPoint = frame.convertFramePointToCameraPoint(center) scheduleOnRN(onSample, cameraPoint, frame.orientation) frame.dispose() @@ -530,10 +538,10 @@ describe('VisionCamera - Coordinates', () => { } }) - // Analyzer coordinates may be reported in the Frame's intended/oriented - // image space, while Frame.convertFramePointToCameraPoint consumes raw - // buffer-space points. The center-only test above cannot catch an - // off-center rectangle drifting after orientation is applied. + // Analyzer coordinates are in the Frame's intended/oriented image space. + // Frame.convertFramePointToCameraPoint must account for the frame + // orientation; the center-only test above cannot catch an off-center + // rectangle drifting after orientation is applied. // See https://github.com/mrousavy/react-native-vision-camera/pull/3878. it('maps oriented Frame rectangles into the same Camera bounds', async () => { const session = await VisionCamera.createCameraSession(false) @@ -554,16 +562,10 @@ describe('VisionCamera - Coordinates', () => { }, ]) - type Bounds = { - left: number - top: number - right: number - bottom: number - } type ProjectionReport = { orientation: string - expected: Bounds - reported: Bounds + orientedCorners: Point[] + recoveredCorners: Point[] } let report: ProjectionReport | undefined const onReport = (r: ProjectionReport) => { @@ -598,38 +600,20 @@ describe('VisionCamera - Coordinates', () => { { x: box.left, y: box.bottom }, ] - const orientedPointToFramePoint = (point: Point): Point => { - switch (frame.orientation) { - case 'right': - return { x: w - point.y, y: point.x } - case 'left': - return { x: point.y, y: h - point.x } - case 'down': - return { x: w - point.x, y: h - point.y } - default: - return point - } - } - const getCameraBounds = (points: Point[]): Bounds => { - const cameraPoints = points.map((point) => - frame.convertFramePointToCameraPoint(point), - ) - const xs = cameraPoints.map((point) => point.x) - const ys = cameraPoints.map((point) => point.y) - return { - left: Math.min(...xs), - top: Math.min(...ys), - right: Math.max(...xs), - bottom: Math.max(...ys), - } - } + // Verify round-trip: oriented → camera → oriented should be identity. + // This catches the bug where orientation is ignored and the conversion + // drifts for off-center points. + const cameraCorners = orientedCorners.map((c) => + frame.convertFramePointToCameraPoint(c), + ) + const recoveredCorners = cameraCorners.map((c) => + frame.convertCameraPointToFramePoint(c), + ) scheduleOnRN(onReport, { orientation: frame.orientation, - expected: getCameraBounds( - orientedCorners.map(orientedPointToFramePoint), - ), - reported: getCameraBounds(orientedCorners), + orientedCorners: orientedCorners, + recoveredCorners: recoveredCorners, }) frame.dispose() }) @@ -650,12 +634,18 @@ describe('VisionCamera - Coordinates', () => { return } - for (const edge of ['left', 'top', 'right', 'bottom'] as const) { - expect(r.reported[edge]).toBeCloseTo(r.expected[edge], 0) + // Each oriented corner, after converting to camera and back, must + // round-trip to within 1 pixel. If orientation is ignored the + // off-center points drift by (roughly) half a frame dimension. + for (let i = 0; i < r.orientedCorners.length; i++) { + const original = r.orientedCorners[i] + const recovered = r.recoveredCorners[i] + expect(recovered.x).toBeCloseTo(original.x, 0) + expect(recovered.y).toBeCloseTo(original.y, 0) } console.log( - `oriented rectangle projection orientation=${r.orientation} expected=${JSON.stringify(r.expected)} reported=${JSON.stringify(r.reported)}`, + `oriented rectangle projection orientation=${r.orientation} corners=${JSON.stringify(r.orientedCorners)} recovered=${JSON.stringify(r.recoveredCorners)}`, ) } finally { runtime.setOnFrameCallback(frameOutput, undefined) diff --git a/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/hybrids/instances/HybridDepthFrame.kt b/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/hybrids/instances/HybridDepthFrame.kt index 40a6159ea4..62c2889b44 100644 --- a/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/hybrids/instances/HybridDepthFrame.kt +++ b/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/hybrids/instances/HybridDepthFrame.kt @@ -112,15 +112,33 @@ class HybridDepthFrame( override fun convertCameraPointToDepthPoint(cameraPoint: Point): Point { val sensorToBuffer = image.imageInfo.sensorToBufferTransformMatrix - return sensorToBuffer.convertPoint(cameraPoint) + val rawFramePoint = sensorToBuffer.convertPoint(cameraPoint) + // Convert from raw buffer space to oriented (display) space + val w = image.width.toDouble() + val h = image.height.toDouble() + return when (orientation) { + CameraOrientation.RIGHT -> Point(rawFramePoint.y, w - rawFramePoint.x) + CameraOrientation.LEFT -> Point(h - rawFramePoint.y, rawFramePoint.x) + CameraOrientation.DOWN -> Point(w - rawFramePoint.x, h - rawFramePoint.y) + CameraOrientation.UP -> rawFramePoint + } } override fun convertDepthPointToCameraPoint(depthPoint: Point): Point { + // Convert from oriented (display) space to raw buffer space + val w = image.width.toDouble() + val h = image.height.toDouble() + val rawFramePoint = when (orientation) { + CameraOrientation.RIGHT -> Point(w - depthPoint.y, depthPoint.x) + CameraOrientation.LEFT -> Point(depthPoint.y, h - depthPoint.x) + CameraOrientation.DOWN -> Point(w - depthPoint.x, h - depthPoint.y) + CameraOrientation.UP -> depthPoint + } val bufferToSensor = Matrix().apply { image.imageInfo.sensorToBufferTransformMatrix.invert(this) } - return bufferToSensor.convertPoint(depthPoint) + return bufferToSensor.convertPoint(rawFramePoint) } override fun toFrame(): HybridFrameSpec { diff --git a/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/hybrids/instances/HybridFrame.kt b/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/hybrids/instances/HybridFrame.kt index 1bfe6a1b9c..901fcdcb2b 100644 --- a/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/hybrids/instances/HybridFrame.kt +++ b/packages/react-native-vision-camera/android/src/main/java/com/margelo/nitro/camera/hybrids/instances/HybridFrame.kt @@ -95,14 +95,32 @@ class HybridFrame( override fun convertCameraPointToFramePoint(cameraPoint: Point): Point { val sensorToBuffer = image.imageInfo.sensorToBufferTransformMatrix - return sensorToBuffer.convertPoint(cameraPoint) + val rawFramePoint = sensorToBuffer.convertPoint(cameraPoint) + // Convert from raw buffer space to oriented (display) space + val w = image.width.toDouble() + val h = image.height.toDouble() + return when (orientation) { + CameraOrientation.RIGHT -> Point(rawFramePoint.y, w - rawFramePoint.x) + CameraOrientation.LEFT -> Point(h - rawFramePoint.y, rawFramePoint.x) + CameraOrientation.DOWN -> Point(w - rawFramePoint.x, h - rawFramePoint.y) + CameraOrientation.UP -> rawFramePoint + } } override fun convertFramePointToCameraPoint(framePoint: Point): Point { + // Convert from oriented (display) space to raw buffer space + val w = image.width.toDouble() + val h = image.height.toDouble() + val rawFramePoint = when (orientation) { + CameraOrientation.RIGHT -> Point(w - framePoint.y, framePoint.x) + CameraOrientation.LEFT -> Point(framePoint.y, h - framePoint.x) + CameraOrientation.DOWN -> Point(w - framePoint.x, h - framePoint.y) + CameraOrientation.UP -> framePoint + } val bufferToSensor = Matrix().apply { image.imageInfo.sensorToBufferTransformMatrix.invert(this) } - return bufferToSensor.convertPoint(framePoint) + return bufferToSensor.convertPoint(rawFramePoint) } }