Skip to content

Commit 6f0d342

Browse files
author
Mantu
committed
feat: add mediapipe poselandmark to detect human body
- Pending implementation on JS
1 parent c9776b1 commit 6f0d342

24 files changed

Lines changed: 14466 additions & 29 deletions

‎README.md‎

Lines changed: 186 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,206 @@
11
# react-native-mediapipe-posedetection
22

3-
PoseDetection using google's mediapipe models using poselandmark
3+
PoseDetection using Google's MediaPipe models with pose landmark detection.
44

5-
## Installation
5+
> **⚠️ New Architecture Required**
6+
>
7+
> This library **only supports** React Native's **New Architecture** (Turbo Modules). You must enable the New Architecture in your app to use this library.
8+
9+
## Requirements
10+
11+
- **React Native:** 0.74.0 or higher
12+
- **New Architecture:** Must be enabled
13+
- **iOS:** iOS 12.0+
14+
- **Android:** API 24+
15+
- **Dependencies:**
16+
- `react-native-vision-camera` (for real-time detection)
17+
- `react-native-worklets-core` (for frame processing)
618

19+
## Installation
720

8-
```sh
21+
```bash
922
npm install react-native-mediapipe-posedetection
23+
# or
24+
yarn add react-native-mediapipe-posedetection
1025
```
1126

27+
### Enable New Architecture
1228

13-
## Usage
29+
If you haven't already enabled the New Architecture in your React Native app:
30+
31+
#### Android
32+
33+
In your `android/gradle.properties`:
34+
35+
```properties
36+
newArchEnabled=true
37+
```
38+
39+
#### iOS
40+
41+
In your `ios/Podfile`:
1442

43+
```ruby
44+
use_frameworks! :linkage => :static
45+
$RNNewArchEnabled = true
46+
```
47+
48+
Then reinstall pods:
49+
50+
```bash
51+
cd ios && pod install
52+
```
53+
54+
### iOS Setup
55+
56+
Add the MediaPipe model to your iOS project:
1557

16-
```js
17-
import { multiply } from 'react-native-mediapipe-posedetection';
58+
1. Download the pose landmarker model
59+
2. Add it to your Xcode project
60+
3. Ensure it's included in "Copy Bundle Resources"
1861

19-
// ...
62+
### Android Setup
63+
64+
The MediaPipe dependencies are automatically included. No additional setup required.
65+
66+
## Usage
67+
68+
### Real-time Pose Detection with Camera
69+
70+
```typescript
71+
import {
72+
usePoseDetection,
73+
RunningMode,
74+
Delegate,
75+
} from 'react-native-mediapipe-posedetection';
76+
import { Camera, useCameraDevice } from 'react-native-vision-camera';
77+
78+
function PoseDetectionScreen() {
79+
const device = useCameraDevice('back');
80+
81+
const poseDetection = usePoseDetection(
82+
{
83+
onResults: (result) => {
84+
console.log('Pose detected:', result.landmarks);
85+
},
86+
onError: (error) => {
87+
console.error('Detection error:', error);
88+
},
89+
},
90+
RunningMode.LIVE_STREAM,
91+
'pose_landmarker_lite.task',
92+
{
93+
numPoses: 1,
94+
minPoseDetectionConfidence: 0.5,
95+
delegate: Delegate.GPU,
96+
}
97+
);
98+
99+
if (!device) return null;
100+
101+
return (
102+
<Camera
103+
device={device}
104+
isActive={true}
105+
frameProcessor={poseDetection.frameProcessor}
106+
onLayout={poseDetection.cameraViewLayoutChangeHandler}
107+
/>
108+
);
109+
}
110+
```
20111

21-
const result = multiply(3, 7);
112+
### Static Image Detection
113+
114+
```typescript
115+
import {
116+
PoseDetectionOnImage,
117+
Delegate,
118+
} from 'react-native-mediapipe-posedetection';
119+
120+
async function detectPoseInImage(imagePath: string) {
121+
const result = await PoseDetectionOnImage(
122+
imagePath,
123+
'pose_landmarker_lite.task',
124+
{
125+
numPoses: 1,
126+
minPoseDetectionConfidence: 0.5,
127+
delegate: Delegate.GPU,
128+
}
129+
);
130+
131+
console.log('Detected poses:', result.landmarks);
132+
}
22133
```
23134

135+
## API Reference
136+
137+
### `usePoseDetection(callbacks, runningMode, model, options)`
138+
139+
Hook for real-time pose detection.
140+
141+
**Parameters:**
142+
143+
- `callbacks`: Object with `onResults` and `onError` handlers
144+
- `runningMode`: `RunningMode.LIVE_STREAM` or `RunningMode.VIDEO`
145+
- `model`: Path to the MediaPipe model file
146+
- `options`: Configuration options (optional)
147+
- `numPoses`: Maximum number of poses to detect (default: 1)
148+
- `minPoseDetectionConfidence`: Minimum confidence for detection (default: 0.5)
149+
- `minPosePresenceConfidence`: Minimum confidence for presence (default: 0.5)
150+
- `minTrackingConfidence`: Minimum confidence for tracking (default: 0.5)
151+
- `delegate`: `Delegate.CPU`, `Delegate.GPU`, or `Delegate.NNAPI` (Android)
152+
- `mirrorMode`: `'no-mirror'`, `'mirror'`, or `'mirror-front-only'`
153+
- `fpsMode`: `'none'` or number (target FPS)
154+
155+
**Returns:** Object with frame processor and camera handlers
156+
157+
### `PoseDetectionOnImage(imagePath, model, options)`
158+
159+
Detect poses in a static image.
160+
161+
**Returns:** Promise resolving to detection results
162+
163+
## Migration from Old Architecture
164+
165+
If you were using a previous version that supported the Old Architecture:
166+
167+
1. **Upgrade React Native** to 0.74.0 or higher
168+
2. **Enable New Architecture** (see instructions above)
169+
3. **Update your app configuration** to remove any Old Architecture compatibility layers
170+
4. **Rebuild your app** completely:
171+
172+
```bash
173+
# iOS
174+
cd ios && pod install && cd ..
175+
176+
# Android
177+
cd android && ./gradlew clean && cd ..
178+
```
179+
180+
The API remains the same, so your application code shouldn't need changes.
181+
182+
## Troubleshooting
183+
184+
### "MediapipePosedetection TurboModule is not available"
185+
186+
**Cause:** New Architecture is not enabled or not properly configured.
187+
188+
**Solution:**
189+
190+
1. Verify `newArchEnabled=true` in `android/gradle.properties`
191+
2. Verify `$RNNewArchEnabled = true` in `ios/Podfile`
192+
3. Clean and rebuild your app
193+
4. Ensure you're using React Native 0.74.0+
194+
195+
### "Failed to create detector"
196+
197+
**Cause:** Model file not found or invalid configuration.
24198

25-
## Contributing
199+
**Solution:**
26200

27-
- [Development workflow](CONTRIBUTING.md#development-workflow)
28-
- [Sending a pull request](CONTRIBUTING.md#sending-a-pull-request)
29-
- [Code of conduct](CODE_OF_CONDUCT.md)
201+
1. Verify the model file path is correct
202+
2. Ensure the model file is included in your app bundle
203+
3. Check that the model file is a valid MediaPipe pose landmarker model
30204

31205
## License
32206

‎android/build.gradle‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,11 @@ apply plugin: "kotlin-android"
2121

2222
apply plugin: "com.facebook.react"
2323

24+
react {
25+
// Enable New Architecture only
26+
// This module is designed for New Architecture (Turbo Modules) only
27+
}
28+
2429
def getExtOrIntegerDefault(name) {
2530
return rootProject.ext.has(name) ? rootProject.ext.get(name) : (project.properties["MediapipePosedetection_" + name]).toInteger()
2631
}
@@ -74,4 +79,9 @@ def kotlin_version = getExtOrDefault("kotlinVersion")
7479
dependencies {
7580
implementation "com.facebook.react:react-android"
7681
implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
82+
implementation 'com.google.mediapipe:tasks-vision:0.10.26'
83+
implementation 'androidx.camera:camera-core:1.3.3'
84+
85+
// VisionCamera for frame processor support
86+
implementation project(':react-native-vision-camera')
7787
}

‎android/src/main/java/com/mediapipeposedetection/MediapipePosedetectionModule.kt‎

Lines changed: 130 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,149 @@
11
package com.mediapipeposedetection
22

3+
import com.facebook.react.bridge.Arguments
4+
import com.facebook.react.bridge.Promise
35
import com.facebook.react.bridge.ReactApplicationContext
6+
import com.facebook.react.bridge.ReactMethod
47
import com.facebook.react.module.annotations.ReactModule
8+
import com.google.mediapipe.tasks.vision.core.RunningMode
9+
import com.mediapipeposedetection.posedetection.PoseDetectorHelper
10+
import com.mediapipeposedetection.posedetection.convertResultBundleToWritableMap
11+
import com.mediapipeposedetection.shared.loadBitmapFromPath
12+
13+
/**
14+
* Global detector map to manage multiple detector instances
15+
*/
16+
object PoseDetectorMap {
17+
internal val detectorMap = mutableMapOf<Int, PoseDetectorHelper>()
18+
}
519

620
@ReactModule(name = MediapipePosedetectionModule.NAME)
721
class MediapipePosedetectionModule(reactContext: ReactApplicationContext) :
822
NativeMediapipePosedetectionSpec(reactContext) {
923

24+
private var nextId = 22 // just not zero
25+
1026
override fun getName(): String {
1127
return NAME
1228
}
1329

14-
// Example method
15-
// See https://reactnative.dev/docs/native-modules-android
16-
override fun multiply(a: Double, b: Double): Double {
17-
return a * b
30+
/**
31+
* Listener for pose detection results and errors
32+
*/
33+
private class PoseDetectorListener(
34+
private val module: MediapipePosedetectionModule,
35+
private val handle: Int
36+
) : PoseDetectorHelper.DetectorListener {
37+
override fun onError(error: String, errorCode: Int) {
38+
module.sendErrorEvent(handle, error, errorCode)
39+
}
40+
41+
override fun onResults(resultBundle: PoseDetectorHelper.ResultBundle) {
42+
module.sendResultsEvent(handle, resultBundle)
43+
}
44+
}
45+
46+
@ReactMethod
47+
override fun createDetector(
48+
numPoses: Double,
49+
minPoseDetectionConfidence: Double,
50+
minPosePresenceConfidence: Double,
51+
minTrackingConfidence: Double,
52+
shouldOutputSegmentationMasks: Boolean,
53+
model: String,
54+
delegate: Double,
55+
runningMode: Double,
56+
promise: Promise
57+
) {
58+
try {
59+
val id = nextId++
60+
val helper = PoseDetectorHelper(
61+
maxNumPoses = numPoses.toInt(),
62+
minPoseDetectionConfidence = minPoseDetectionConfidence.toFloat(),
63+
minPosePresenceConfidence = minPosePresenceConfidence.toFloat(),
64+
minPoseTrackingConfidence = minTrackingConfidence.toFloat(),
65+
shouldOutputSegmentationMasks = shouldOutputSegmentationMasks,
66+
currentDelegate = delegate.toInt(),
67+
currentModel = model,
68+
runningMode = enumValues<RunningMode>().first { it.ordinal == runningMode.toInt() },
69+
context = reactApplicationContext.applicationContext,
70+
poseDetectorListener = PoseDetectorListener(this, id)
71+
)
72+
PoseDetectorMap.detectorMap[id] = helper
73+
promise.resolve(id.toDouble())
74+
} catch (e: Exception) {
75+
promise.reject("CREATE_DETECTOR_ERROR", "Failed to create detector: ${e.message}", e)
76+
}
77+
}
78+
79+
@ReactMethod
80+
override fun releaseDetector(handle: Double, promise: Promise) {
81+
try {
82+
val entry = PoseDetectorMap.detectorMap[handle.toInt()]
83+
if (entry != null) {
84+
entry.clearPoseLandmarker()
85+
PoseDetectorMap.detectorMap.remove(handle.toInt())
86+
}
87+
promise.resolve(true)
88+
} catch (e: Exception) {
89+
promise.reject("RELEASE_DETECTOR_ERROR", "Failed to release detector: ${e.message}", e)
90+
}
91+
}
92+
93+
@ReactMethod
94+
override fun detectOnImage(
95+
imagePath: String,
96+
numPoses: Double,
97+
minPoseDetectionConfidence: Double,
98+
minPosePresenceConfidence: Double,
99+
minTrackingConfidence: Double,
100+
shouldOutputSegmentationMasks: Boolean,
101+
model: String,
102+
delegate: Double,
103+
promise: Promise
104+
) {
105+
try {
106+
val helper = PoseDetectorHelper(
107+
maxNumPoses = numPoses.toInt(),
108+
minPoseDetectionConfidence = minPoseDetectionConfidence.toFloat(),
109+
minPosePresenceConfidence = minPosePresenceConfidence.toFloat(),
110+
minPoseTrackingConfidence = minTrackingConfidence.toFloat(),
111+
shouldOutputSegmentationMasks = shouldOutputSegmentationMasks,
112+
currentDelegate = delegate.toInt(),
113+
currentModel = model,
114+
runningMode = RunningMode.IMAGE,
115+
context = reactApplicationContext.applicationContext,
116+
poseDetectorListener = PoseDetectorListener(this, 0)
117+
)
118+
val bundle = helper.detectImage(loadBitmapFromPath(imagePath))
119+
val resultArgs = convertResultBundleToWritableMap(bundle)
120+
121+
promise.resolve(resultArgs)
122+
} catch (e: Exception) {
123+
promise.reject("DETECT_IMAGE_ERROR", "Failed to detect on image: ${e.message}", e)
124+
}
125+
}
126+
127+
private fun sendErrorEvent(handle: Int, message: String, code: Int) {
128+
android.util.Log.d(TAG, "Sending error event: $message")
129+
val errorArgs = Arguments.createMap()
130+
errorArgs.putInt("handle", handle)
131+
errorArgs.putString("message", message)
132+
errorArgs.putInt("code", code)
133+
134+
emitOnError(errorArgs)
135+
}
136+
137+
private fun sendResultsEvent(handle: Int, bundle: PoseDetectorHelper.ResultBundle) {
138+
android.util.Log.d(TAG, "Sending results event for handle: $handle")
139+
val resultArgs = convertResultBundleToWritableMap(bundle)
140+
resultArgs.putInt("handle", handle)
141+
142+
emitOnResults(resultArgs)
18143
}
19144

20145
companion object {
21146
const val NAME = "MediapipePosedetection"
147+
private const val TAG = "MediapipePosedetectionModule"
22148
}
23149
}

0 commit comments

Comments
 (0)