diff --git a/README.md b/README.md index f4a7840..1aac9f8 100644 --- a/README.md +++ b/README.md @@ -37,12 +37,12 @@ mobile-ink is currently used in production in MathNotes: https://apps.apple.com/ | Area | Current support | | --- | --- | | iOS Apple Pencil drawing | Used in production | -| Native rendering | Custom `MTKView` backed by C++ Skia/Metal | +| Native rendering | Custom iOS `MTKView` and Android `TextureView` backed by the shared C++ Skia engine | | Continuous notebooks | Fixed native engine pool with momentum scroll and pinch zoom | | Tools | Pen, highlighter, crayon, calligraphy, eraser, selection, and shape recognition | | Serialization | JSON notebook payloads plus native page load/save/export helpers | | Example app | Expo dev-client app with blank continuous notebook, tools, selection, save/reload, and zoom | -| Android | Not supported yet | +| Android | V1 native drawing support with GPU-backed Ganesh rendering, pooled pages, previews, save/reload, eraser, selection, and PDF backgrounds | | Expo Go | Not supported because this package includes native code | ## Demos @@ -68,7 +68,7 @@ npm install @mathnotes/mobile-ink \ cd ios && pod install ``` -For Expo apps, use a dev client or prebuild. Expo Go cannot load this native module. +For Expo apps, use a dev client or prebuild. Expo Go cannot load this native module. Android builds also need a configured Android SDK, NDK, and CMake toolchain. Your app Babel config must include the Reanimated/Worklets plugin expected by your React Native/Reanimated version. For Expo SDK 54/Reanimated 4: @@ -132,6 +132,8 @@ export function Notebook() { The `example/` folder is an Expo dev-client app that exercises the reusable canvas stack, not a MathNotes screen. It demos the full continuous canvas path: pencil drawing with finger navigation by default, optional draw-with-finger mode, pinch zoom, momentum scroll, engine-pool page assignment, MathNotes-style one-page trailing blank growth, tools, selection, and local save/reload on a blank page background. +Expo Go cannot run the example because this package includes native Kotlin, C++, and iOS code. Use a dev-client build: + ```sh cd example npm install @@ -144,6 +146,21 @@ For a simulator: npx expo run:ios ``` +For Android: + +```sh +npx expo run:android +``` + +The first Android build compiles the shared C++ drawing engine and can take a while. If the generated native project is stale after Android native changes, run: + +```sh +npx expo prebuild --platform android --clean +npx expo run:android +``` + +The Android example runs the drawing canvas path. The benchmark screen and CPU/Ganesh backend toggle are currently iOS-only. See [example/README.md](example/README.md) for Android prerequisites, Metro/dev-client commands, and smoke checks. + ## Documentation - [Architecture](docs/architecture.md) @@ -152,14 +169,14 @@ npx expo run:ios ## Roadmap -Near-term work is focused on making the public package easier to adopt and easier to contribute to, and achieving Android parity: +Near-term work is focused on making the public package easier to adopt and easier to contribute to, and hardening Android v1: - Improve install and troubleshooting docs for React Native and Expo dev-client apps. - Add more integration recipes for save/load, tool switching, and app-owned storage. - Tighten selection transform performance for large stroke groups. - Improve edge-case zoom behavior near page and canvas boundaries. - Continue hardening the example app as a small regression harness. -- Complete Android parity with iOS. We started this but it is still not quite there. +- Add the Android native benchmark runner and expose Android benchmark controls in the example app. ## Development @@ -173,6 +190,7 @@ npm pack --dry-run --ignore-scripts npm ci --prefix example npm run test:example:typecheck npm run test:example:export:ios +npm run test:example:export:android ``` `npm run build` creates `lib/` for npm packaging. The React Native entry still points at `src/index.ts` so Metro can transform worklet directives with the consuming app's Babel config. diff --git a/android/.gitignore b/android/.gitignore new file mode 100644 index 0000000..6bd26ee --- /dev/null +++ b/android/.gitignore @@ -0,0 +1,2 @@ +/.cxx/ +/build/ diff --git a/android/build.gradle b/android/build.gradle new file mode 100644 index 0000000..b5793ef --- /dev/null +++ b/android/build.gradle @@ -0,0 +1,92 @@ +buildscript { + def kotlinVersion = rootProject.ext.has("kotlinVersion") + ? rootProject.ext.get("kotlinVersion") + : "2.1.20" + + repositories { + google() + mavenCentral() + } + + dependencies { + classpath("com.android.tools.build:gradle:8.10.1") + classpath("org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlinVersion") + } +} + +def safeExtGet(prop, fallback) { + rootProject.ext.has(prop) ? rootProject.ext.get(prop) : fallback +} + +static def findNodeModules(baseDir) { + def basePath = baseDir.toPath().normalize() + while (basePath != null) { + def nodeModulesPath = basePath.resolve("node_modules").toFile() + def reactNativePath = new File(nodeModulesPath, "react-native") + if (nodeModulesPath.exists() && reactNativePath.exists()) { + return nodeModulesPath.absolutePath + } + basePath = basePath.getParent() + } + throw new GradleException("@mathnotes/mobile-ink: failed to find node_modules") +} + +def reactNativeArchitectures() { + def value = project.getProperties().get("reactNativeArchitectures") + return value ? value.split(",") : ["armeabi-v7a", "x86", "x86_64", "arm64-v8a"] +} + +apply plugin: "com.android.library" +apply plugin: "kotlin-android" + +def nodeModules = findNodeModules(projectDir) + +android { + namespace "com.mathnotes.mobileink" + compileSdkVersion safeExtGet("compileSdkVersion", 35) + + if (rootProject.ext.has("ndkPath")) { + ndkPath rootProject.ext.ndkPath + } + if (rootProject.ext.has("ndkVersion")) { + ndkVersion rootProject.ext.ndkVersion + } + + defaultConfig { + minSdkVersion safeExtGet("minSdkVersion", 24) + targetSdkVersion safeExtGet("targetSdkVersion", 35) + + externalNativeBuild { + cmake { + cppFlags "-std=c++17", "-frtti", "-fexceptions", "-DSK_BUILD_FOR_ANDROID", "-DON_ANDROID", "-DONANDROID" + arguments "-DNODE_MODULES_DIR=${nodeModules}", + "-DANDROID_STL=c++_shared", + "-DANDROID_SUPPORT_FLEXIBLE_PAGE_SIZES=ON" + abiFilters(*reactNativeArchitectures()) + } + } + } + + compileOptions { + sourceCompatibility JavaVersion.VERSION_1_8 + targetCompatibility JavaVersion.VERSION_1_8 + } + + externalNativeBuild { + cmake { + path "src/main/cpp/CMakeLists.txt" + version "3.22.1" + } + } + + packagingOptions { + excludes = [ + "**/libc++_shared.so", + "META-INF/**" + ] + } +} + +dependencies { + implementation "com.facebook.react:react-android" +} diff --git a/android/src/main/AndroidManifest.xml b/android/src/main/AndroidManifest.xml new file mode 100644 index 0000000..94cbbcf --- /dev/null +++ b/android/src/main/AndroidManifest.xml @@ -0,0 +1 @@ + diff --git a/android/src/main/cpp/CMakeLists.txt b/android/src/main/cpp/CMakeLists.txt new file mode 100644 index 0000000..e17d059 --- /dev/null +++ b/android/src/main/cpp/CMakeLists.txt @@ -0,0 +1,60 @@ +cmake_minimum_required(VERSION 3.13) +project(mobileink) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +# Node modules path passed from Gradle +set(NODE_MODULES_DIR ${NODE_MODULES_DIR}) + +# Skia paths from react-native-skia +set(SKIA_DIR "${NODE_MODULES_DIR}/@shopify/react-native-skia") +set(SKIA_INCLUDE_DIR "${SKIA_DIR}/cpp/skia") +set(SKIA_LIB_DIR "${SKIA_DIR}/libs/android/${ANDROID_ABI}") + +# Shared C++ drawing engine from this package root. +set(NATIVE_DRAWING_CPP_DIR "${CMAKE_CURRENT_LIST_DIR}/../../../../cpp") + +message(STATUS "mobileink: NODE_MODULES_DIR=${NODE_MODULES_DIR}") +message(STATUS "mobileink: SKIA_DIR=${SKIA_DIR}") +message(STATUS "mobileink: NATIVE_DRAWING_CPP_DIR=${NATIVE_DRAWING_CPP_DIR}") +message(STATUS "mobileink: ANDROID_ABI=${ANDROID_ABI}") + +# Include directories +include_directories( + ${SKIA_INCLUDE_DIR} + ${SKIA_INCLUDE_DIR}/include + ${SKIA_INCLUDE_DIR}/modules/pathops/include + ${NATIVE_DRAWING_CPP_DIR} +) + +file(GLOB NATIVE_DRAWING_SOURCES CONFIGURE_DEPENDS + "${NATIVE_DRAWING_CPP_DIR}/*.cpp" +) + +# Add source files +add_library( + mobileink + SHARED + ${NATIVE_DRAWING_SOURCES} + jni_bridge.cpp +) + +# Link Skia library +add_library(skia STATIC IMPORTED) +set_target_properties(skia PROPERTIES IMPORTED_LOCATION "${SKIA_LIB_DIR}/libskia.a") + +add_library(pathops STATIC IMPORTED) +set_target_properties(pathops PROPERTIES IMPORTED_LOCATION "${SKIA_LIB_DIR}/libpathops.a") + +# Link libraries +target_link_libraries( + mobileink + skia + pathops + android + log + GLESv2 + EGL + jnigraphics +) diff --git a/android/src/main/cpp/jni_bridge.cpp b/android/src/main/cpp/jni_bridge.cpp new file mode 100644 index 0000000..39990d7 --- /dev/null +++ b/android/src/main/cpp/jni_bridge.cpp @@ -0,0 +1,1160 @@ +#include +#include +#include +#include +#include "SkiaDrawingEngine.h" +#include "DrawingSerialization.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define LOG_TAG "MobileInk" +#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR, LOG_TAG, __VA_ARGS__) + +using namespace nativedrawing; + +namespace { + +struct PageStrokeSegment { + int pageIndex = 0; + std::vector points; +}; + +int clampPageIndexForY(float y, int pageCount, float pageHeight) { + if (pageCount <= 0 || pageHeight <= 0.0f) { + return 0; + } + + const int unclamped = static_cast(std::floor(y / pageHeight)); + return std::max(0, std::min(pageCount - 1, unclamped)); +} + +Point interpolatePointAtY(const Point& start, const Point& end, float targetY) { + const float denominator = end.y - start.y; + const float rawT = std::fabs(denominator) < 0.0001f + ? 0.0f + : (targetY - start.y) / denominator; + const float t = std::max(0.0f, std::min(1.0f, rawT)); + + Point interpolated; + interpolated.x = start.x + (end.x - start.x) * t; + interpolated.y = targetY; + interpolated.pressure = start.pressure + (end.pressure - start.pressure) * t; + interpolated.azimuthAngle = start.azimuthAngle + (end.azimuthAngle - start.azimuthAngle) * t; + interpolated.altitude = start.altitude + (end.altitude - start.altitude) * t; + interpolated.calculatedWidth = start.calculatedWidth + (end.calculatedWidth - start.calculatedWidth) * t; + interpolated.timestamp = start.timestamp + static_cast((end.timestamp - start.timestamp) * t); + return interpolated; +} + +bool deserializeDrawingBytes( + const uint8_t* data, + int size, + DrawingSerialization& serializer, + std::vector& strokes +) { + if (!data || size <= 0) { + strokes.clear(); + return true; + } + + const std::vector buffer(data, data + size); + return serializer.deserialize(buffer, strokes); +} + +sk_sp makeRedBlueSwapFilter() { + float redBlueSwapMatrix[20] = { + 0, 0, 1, 0, 0, + 0, 1, 0, 0, 0, + 1, 0, 0, 0, 0, + 0, 0, 0, 1, 0 + }; + + return SkColorFilters::Matrix(redBlueSwapMatrix); +} + +sk_sp makeRedBlueSwappedImage(const sk_sp& image) { + if (!image) { + return nullptr; + } + + SkImageInfo info = SkImageInfo::MakeN32Premul(image->width(), image->height()); + sk_sp surface = SkSurfaces::Raster(info); + if (!surface) { + return image; + } + + SkPaint outputPaint; + outputPaint.setColorFilter(makeRedBlueSwapFilter()); + surface->getCanvas()->drawImage( + image, + 0, + 0, + SkSamplingOptions(), + &outputPaint + ); + return surface->makeImageSnapshot(); +} + +void renderWithDisplayColorOrdering(SkiaDrawingEngine* engine, SkCanvas* canvas) { + if (!engine || !canvas) { + return; + } + + SkPaint outputPaint; + outputPaint.setColorFilter(makeRedBlueSwapFilter()); + canvas->saveLayer(nullptr, &outputPaint); + engine->render(canvas); + canvas->restore(); +} + +void translateStrokeInPlace(Stroke& stroke, float deltaY, size_t affectedStrokeOffset) { + for (auto& point : stroke.points) { + point.y += deltaY; + } + + for (auto& circle : stroke.erasedBy) { + circle.y += deltaY; + } + + if (stroke.isEraser && affectedStrokeOffset > 0) { + std::unordered_set translatedIndices; + for (size_t index : stroke.affectedStrokeIndices) { + translatedIndices.insert(index + affectedStrokeOffset); + } + stroke.affectedStrokeIndices = std::move(translatedIndices); + } + + stroke.path.reset(); + stroke.pathLength = 0.0f; + stroke.cachedEraserPath.reset(); + stroke.cachedEraserCount = 0; + stroke.cachedHasVisiblePoints = true; +} + +std::vector splitStrokeAcrossPages( + const Stroke& stroke, + int pageCount, + float pageHeight +) { + std::vector segments; + if (stroke.points.empty() || pageCount <= 0 || pageHeight <= 0.0f) { + return segments; + } + + const int firstPageIndex = clampPageIndexForY(stroke.points.front().y, pageCount, pageHeight); + segments.push_back(PageStrokeSegment{ firstPageIndex, { stroke.points.front() } }); + + for (size_t index = 1; index < stroke.points.size(); ++index) { + const Point& previousPoint = stroke.points[index - 1]; + const Point& currentPoint = stroke.points[index]; + const int previousPageIndex = clampPageIndexForY(previousPoint.y, pageCount, pageHeight); + const int currentPageIndex = clampPageIndexForY(currentPoint.y, pageCount, pageHeight); + + if (previousPageIndex == currentPageIndex || std::fabs(currentPoint.y - previousPoint.y) < 0.0001f) { + segments.back().points.push_back(currentPoint); + continue; + } + + int traversedPageIndex = previousPageIndex; + const int direction = currentPageIndex > previousPageIndex ? 1 : -1; + + while (traversedPageIndex != currentPageIndex) { + const float boundaryY = direction > 0 + ? static_cast(traversedPageIndex + 1) * pageHeight + : static_cast(traversedPageIndex) * pageHeight; + + Point boundaryPoint = interpolatePointAtY(previousPoint, currentPoint, boundaryY); + segments.back().points.push_back(boundaryPoint); + + traversedPageIndex += direction; + PageStrokeSegment nextSegment; + nextSegment.pageIndex = traversedPageIndex; + nextSegment.points.push_back(boundaryPoint); + segments.push_back(std::move(nextSegment)); + } + + segments.back().points.push_back(currentPoint); + } + + return segments; +} + +Stroke makePageLocalStroke(const Stroke& originalStroke, const PageStrokeSegment& segment, float pageHeight) { + Stroke pageStroke = originalStroke; + pageStroke.points.clear(); + pageStroke.points.reserve(segment.points.size()); + pageStroke.affectedStrokeIndices.clear(); + pageStroke.erasedBy.clear(); + pageStroke.path.reset(); + pageStroke.pathLength = 0.0f; + pageStroke.cachedEraserPath.reset(); + pageStroke.cachedEraserCount = 0; + pageStroke.cachedHasVisiblePoints = true; + + const float pageTop = static_cast(segment.pageIndex) * pageHeight; + const float pageBottom = pageTop + pageHeight; + + for (const auto& point : segment.points) { + Point localPoint = point; + localPoint.y -= pageTop; + pageStroke.points.push_back(localPoint); + } + + for (const auto& circle : originalStroke.erasedBy) { + if ((circle.y + circle.radius) < pageTop || (circle.y - circle.radius) > pageBottom) { + continue; + } + + EraserCircle localCircle = circle; + localCircle.y -= pageTop; + pageStroke.erasedBy.push_back(localCircle); + } + + return pageStroke; +} + +std::vector composeContinuousWindowBytes( + const std::vector>& pageData, + float pageHeight +) { + if (pageData.empty() || pageHeight <= 0.0f) { + return {}; + } + + DrawingSerialization serializer; + std::vector combinedStrokes; + + for (size_t pageIndex = 0; pageIndex < pageData.size(); ++pageIndex) { + std::vector pageStrokes; + const auto& data = pageData[pageIndex]; + if (!deserializeDrawingBytes(data.data(), static_cast(data.size()), serializer, pageStrokes)) { + continue; + } + + const size_t affectedStrokeOffset = combinedStrokes.size(); + const float yOffset = static_cast(pageIndex) * pageHeight; + for (auto& stroke : pageStrokes) { + translateStrokeInPlace(stroke, yOffset, affectedStrokeOffset); + combinedStrokes.push_back(std::move(stroke)); + } + } + + return combinedStrokes.empty() ? std::vector() : serializer.serialize(combinedStrokes); +} + +std::vector> decomposeContinuousWindowBytes( + const uint8_t* windowData, + int windowDataSize, + int pageCount, + float pageHeight +) { + std::vector> result(static_cast(std::max(0, pageCount))); + if (pageCount <= 0 || pageHeight <= 0.0f) { + return result; + } + + if (!windowData || windowDataSize <= 0) { + return result; + } + + DrawingSerialization serializer; + std::vector combinedStrokes; + if (!deserializeDrawingBytes(windowData, windowDataSize, serializer, combinedStrokes)) { + result.clear(); + return result; + } + + std::vector> pageStrokes(static_cast(pageCount)); + std::vector>> outputIndexMap(combinedStrokes.size()); + + for (size_t originalStrokeIndex = 0; originalStrokeIndex < combinedStrokes.size(); ++originalStrokeIndex) { + const Stroke& originalStroke = combinedStrokes[originalStrokeIndex]; + std::vector segments = splitStrokeAcrossPages(originalStroke, pageCount, pageHeight); + for (const auto& segment : segments) { + Stroke pageLocalStroke = makePageLocalStroke(originalStroke, segment, pageHeight); + pageStrokes[segment.pageIndex].push_back(std::move(pageLocalStroke)); + outputIndexMap[originalStrokeIndex][segment.pageIndex].push_back( + pageStrokes[segment.pageIndex].size() - 1 + ); + } + } + + for (size_t originalStrokeIndex = 0; originalStrokeIndex < combinedStrokes.size(); ++originalStrokeIndex) { + const Stroke& originalStroke = combinedStrokes[originalStrokeIndex]; + if (!originalStroke.isEraser) { + continue; + } + + for (const auto& pageEntry : outputIndexMap[originalStrokeIndex]) { + const int pageIndex = pageEntry.first; + const std::vector& eraserSegmentIndices = pageEntry.second; + std::unordered_set remappedAffectedIndices; + for (size_t affectedOriginalIndex : originalStroke.affectedStrokeIndices) { + if (affectedOriginalIndex >= outputIndexMap.size()) { + continue; + } + + const auto affectedSegmentsIt = outputIndexMap[affectedOriginalIndex].find(pageIndex); + if (affectedSegmentsIt == outputIndexMap[affectedOriginalIndex].end()) { + continue; + } + + remappedAffectedIndices.insert( + affectedSegmentsIt->second.begin(), + affectedSegmentsIt->second.end() + ); + } + + for (size_t eraserSegmentIndex : eraserSegmentIndices) { + if (eraserSegmentIndex < pageStrokes[pageIndex].size()) { + pageStrokes[pageIndex][eraserSegmentIndex].affectedStrokeIndices = remappedAffectedIndices; + } + } + } + } + + for (int pageIndex = 0; pageIndex < pageCount; ++pageIndex) { + if (!pageStrokes[pageIndex].empty()) { + result[pageIndex] = serializer.serialize(pageStrokes[pageIndex]); + } + } + + return result; +} + +} // namespace + +// DrawingContext owns the shared drawing model plus the active Ganesh render surface. +struct DrawingContext { + std::unique_ptr engine; + sk_sp ganeshContext; + sk_sp ganeshSurface; + int ganeshSurfaceWidth = 0; + int ganeshSurfaceHeight = 0; + int ganeshSamples = -1; + int ganeshStencil = -1; + + DrawingContext(int w, int h) { + engine = std::make_unique(w, h); + } + + ~DrawingContext() { + ganeshSurface = nullptr; + if (ganeshContext) { + ganeshContext->abandonContext(); + ganeshContext = nullptr; + } + } +}; + +extern "C" { + +// Engine lifecycle +JNIEXPORT jlong JNICALL +Java_com_mathnotes_mobileink_MobileInkCanvasView_createDrawingEngine( + JNIEnv* env, jobject obj, jint width, jint height) { + + auto* ctx = new DrawingContext(width, height); + return reinterpret_cast(ctx); +} + +JNIEXPORT void JNICALL +Java_com_mathnotes_mobileink_MobileInkCanvasView_destroyDrawingEngine( + JNIEnv* env, jobject obj, jlong contextPtr) { + + auto* ctx = reinterpret_cast(contextPtr); + delete ctx; +} + +// Touch handling with stylus support (pressure, azimuth, altitude) +JNIEXPORT void JNICALL +Java_com_mathnotes_mobileink_MobileInkCanvasView_touchBegan( + JNIEnv* env, jobject obj, jlong contextPtr, + jfloat x, jfloat y, jfloat pressure, jfloat azimuth, jfloat altitude, + jlong timestamp, jboolean isStylusInput) { + + auto* ctx = reinterpret_cast(contextPtr); + if (ctx && ctx->engine) { + ctx->engine->touchBegan(x, y, pressure, azimuth, altitude, static_cast(timestamp), isStylusInput == JNI_TRUE); + } else { + LOGE("touchBegan: context or engine is null! ctx=%p", ctx); + } +} + +JNIEXPORT void JNICALL +Java_com_mathnotes_mobileink_MobileInkCanvasView_touchMoved( + JNIEnv* env, jobject obj, jlong contextPtr, + jfloat x, jfloat y, jfloat pressure, jfloat azimuth, jfloat altitude, + jlong timestamp, jboolean isStylusInput) { + + auto* ctx = reinterpret_cast(contextPtr); + if (ctx && ctx->engine) { + ctx->engine->touchMoved(x, y, pressure, azimuth, altitude, static_cast(timestamp), isStylusInput == JNI_TRUE); + } else { + LOGE("touchMoved: context or engine is null!"); + } +} + +JNIEXPORT void JNICALL +Java_com_mathnotes_mobileink_MobileInkCanvasView_touchEnded(JNIEnv* env, jobject obj, jlong contextPtr, jlong timestamp) { + auto* ctx = reinterpret_cast(contextPtr); + if (ctx && ctx->engine) { + ctx->engine->touchEnded(static_cast(timestamp)); + } else { + LOGE("touchEnded: context or engine is null!"); + } +} + +JNIEXPORT jboolean JNICALL +Java_com_mathnotes_mobileink_MobileInkCanvasView_updateHoldShapePreview( + JNIEnv* env, jobject obj, jlong contextPtr, jlong timestamp) { + + auto* ctx = reinterpret_cast(contextPtr); + if (ctx && ctx->engine) { + return ctx->engine->updateHoldShapePreview(static_cast(timestamp)) ? JNI_TRUE : JNI_FALSE; + } + + return JNI_FALSE; +} + +// Canvas operations +JNIEXPORT void JNICALL +Java_com_mathnotes_mobileink_MobileInkCanvasView_clearCanvas(JNIEnv* env, jobject obj, jlong contextPtr) { + auto* ctx = reinterpret_cast(contextPtr); + if (ctx && ctx->engine) { + ctx->engine->clear(); + } +} + +JNIEXPORT void JNICALL +Java_com_mathnotes_mobileink_MobileInkCanvasView_undoStroke(JNIEnv* env, jobject obj, jlong contextPtr) { + auto* ctx = reinterpret_cast(contextPtr); + if (ctx && ctx->engine) { + ctx->engine->undo(); + } +} + +JNIEXPORT void JNICALL +Java_com_mathnotes_mobileink_MobileInkCanvasView_redoStroke(JNIEnv* env, jobject obj, jlong contextPtr) { + auto* ctx = reinterpret_cast(contextPtr); + if (ctx && ctx->engine) { + ctx->engine->redo(); + } +} + +// Tool settings +JNIEXPORT void JNICALL +Java_com_mathnotes_mobileink_MobileInkCanvasView_setStrokeColor( + JNIEnv* env, jobject obj, jlong contextPtr, jint color) { + + auto* ctx = reinterpret_cast(contextPtr); + if (ctx && ctx->engine) { + // Android color is ARGB, Skia uses ARGB too but we need SkColor format + ctx->engine->setStrokeColor(static_cast(color)); + } +} + +JNIEXPORT void JNICALL +Java_com_mathnotes_mobileink_MobileInkCanvasView_setStrokeWidth( + JNIEnv* env, jobject obj, jlong contextPtr, jfloat width) { + + auto* ctx = reinterpret_cast(contextPtr); + if (ctx && ctx->engine) { + ctx->engine->setStrokeWidth(width); + } +} + +JNIEXPORT void JNICALL +Java_com_mathnotes_mobileink_MobileInkCanvasView_setTool( + JNIEnv* env, jobject obj, jlong contextPtr, jstring toolType) { + + auto* ctx = reinterpret_cast(contextPtr); + if (ctx && ctx->engine) { + const char* toolStr = env->GetStringUTFChars(toolType, nullptr); + ctx->engine->setTool(toolStr); + env->ReleaseStringUTFChars(toolType, toolStr); + } +} + +JNIEXPORT void JNICALL +Java_com_mathnotes_mobileink_MobileInkCanvasView_setToolWithParams( + JNIEnv* env, jobject obj, jlong contextPtr, + jstring toolType, jfloat width, jint color, jstring eraserMode) { + + auto* ctx = reinterpret_cast(contextPtr); + if (ctx && ctx->engine) { + const char* toolStr = env->GetStringUTFChars(toolType, nullptr); + const char* eraserStr = eraserMode ? env->GetStringUTFChars(eraserMode, nullptr) : ""; + + ctx->engine->setToolWithParams(toolStr, width, static_cast(color), eraserStr); + + env->ReleaseStringUTFChars(toolType, toolStr); + if (eraserMode) { + env->ReleaseStringUTFChars(eraserMode, eraserStr); + } + } +} + +// Eraser cursor for pixel eraser visualization +JNIEXPORT void JNICALL +Java_com_mathnotes_mobileink_MobileInkCanvasView_setEraserCursor( + JNIEnv* env, jobject obj, jlong contextPtr, + jfloat x, jfloat y, jfloat radius, jboolean visible) { + + auto* ctx = reinterpret_cast(contextPtr); + if (ctx && ctx->engine) { + ctx->engine->setEraserCursor(x, y, radius, visible == JNI_TRUE); + } +} + +// Background type for pattern rendering +JNIEXPORT void JNICALL +Java_com_mathnotes_mobileink_MobileInkCanvasView_setBackgroundType( + JNIEnv* env, jobject obj, jlong contextPtr, jstring backgroundType) { + + auto* ctx = reinterpret_cast(contextPtr); + if (ctx && ctx->engine) { + const char* typeStr = env->GetStringUTFChars(backgroundType, nullptr); + ctx->engine->setBackgroundType(typeStr); + env->ReleaseStringUTFChars(backgroundType, typeStr); + } +} + +// PDF background bitmap - render PDF in Kotlin, pass to C++ as SkImage +JNIEXPORT void JNICALL +Java_com_mathnotes_mobileink_MobileInkCanvasView_setPdfBackgroundBitmap( + JNIEnv* env, jobject obj, jlong contextPtr, jobject bitmap) { + + auto* ctx = reinterpret_cast(contextPtr); + if (!ctx || !ctx->engine) { + LOGE("setPdfBackgroundBitmap: context or engine is null"); + return; + } + + if (bitmap == nullptr) { + ctx->engine->setPdfBackgroundImage(nullptr); + return; + } + + // Get bitmap info + AndroidBitmapInfo info; + if (AndroidBitmap_getInfo(env, bitmap, &info) != ANDROID_BITMAP_RESULT_SUCCESS) { + LOGE("setPdfBackgroundBitmap: failed to get bitmap info"); + return; + } + + // Lock pixels + void* pixels = nullptr; + if (AndroidBitmap_lockPixels(env, bitmap, &pixels) != ANDROID_BITMAP_RESULT_SUCCESS) { + LOGE("setPdfBackgroundBitmap: failed to lock pixels"); + return; + } + + // Create SkImage from bitmap pixels + // Android ARGB_8888 is compatible with Skia N32 (BGRA on little-endian) + SkImageInfo skInfo = SkImageInfo::MakeN32Premul(info.width, info.height); + + // Copy pixel data since we need to unlock the bitmap + size_t dataSize = info.height * info.stride; + sk_sp data = SkData::MakeWithCopy(pixels, dataSize); + + AndroidBitmap_unlockPixels(env, bitmap); + + // Create the image from the copied data + sk_sp image = SkImages::RasterFromData(skInfo, data, info.stride); + + if (image) { + ctx->engine->setPdfBackgroundImage(makeRedBlueSwappedImage(image)); + } else { + LOGE("setPdfBackgroundBitmap: failed to create SkImage"); + } +} + +// Selection operations +JNIEXPORT jboolean JNICALL +Java_com_mathnotes_mobileink_MobileInkCanvasView_selectStrokeAt( + JNIEnv* env, jobject obj, jlong contextPtr, jfloat x, jfloat y) { + + auto* ctx = reinterpret_cast(contextPtr); + if (ctx && ctx->engine) { + return ctx->engine->selectStrokeAt(x, y) ? JNI_TRUE : JNI_FALSE; + } + return JNI_FALSE; +} + +JNIEXPORT jboolean JNICALL +Java_com_mathnotes_mobileink_MobileInkCanvasView_selectShapeStrokeAt( + JNIEnv* env, jobject obj, jlong contextPtr, jfloat x, jfloat y) { + + auto* ctx = reinterpret_cast(contextPtr); + if (ctx && ctx->engine) { + return ctx->engine->selectShapeStrokeAt(x, y) ? JNI_TRUE : JNI_FALSE; + } + return JNI_FALSE; +} + +JNIEXPORT void JNICALL +Java_com_mathnotes_mobileink_MobileInkCanvasView_clearSelection(JNIEnv* env, jobject obj, jlong contextPtr) { + auto* ctx = reinterpret_cast(contextPtr); + if (ctx && ctx->engine) { + ctx->engine->clearSelection(); + } +} + +JNIEXPORT void JNICALL +Java_com_mathnotes_mobileink_MobileInkCanvasView_deleteSelection(JNIEnv* env, jobject obj, jlong contextPtr) { + auto* ctx = reinterpret_cast(contextPtr); + if (ctx && ctx->engine) { + ctx->engine->deleteSelection(); + } +} + +JNIEXPORT void JNICALL +Java_com_mathnotes_mobileink_MobileInkCanvasView_copySelection(JNIEnv* env, jobject obj, jlong contextPtr) { + auto* ctx = reinterpret_cast(contextPtr); + if (ctx && ctx->engine) { + ctx->engine->copySelection(); + } +} + +JNIEXPORT void JNICALL +Java_com_mathnotes_mobileink_MobileInkCanvasView_pasteSelection( + JNIEnv* env, jobject obj, jlong contextPtr, jfloat offsetX, jfloat offsetY) { + + auto* ctx = reinterpret_cast(contextPtr); + if (ctx && ctx->engine) { + ctx->engine->pasteSelection(offsetX, offsetY); + } +} + +JNIEXPORT void JNICALL +Java_com_mathnotes_mobileink_MobileInkCanvasView_moveSelection( + JNIEnv* env, jobject obj, jlong contextPtr, jfloat dx, jfloat dy) { + + auto* ctx = reinterpret_cast(contextPtr); + if (ctx && ctx->engine) { + ctx->engine->moveSelection(dx, dy); + } +} + +JNIEXPORT void JNICALL +Java_com_mathnotes_mobileink_MobileInkCanvasView_finalizeMove(JNIEnv* env, jobject obj, jlong contextPtr) { + auto* ctx = reinterpret_cast(contextPtr); + if (ctx && ctx->engine) { + ctx->engine->finalizeMove(); + } +} + +JNIEXPORT void JNICALL +Java_com_mathnotes_mobileink_MobileInkCanvasView_beginSelectionTransform( + JNIEnv* env, jobject obj, jlong contextPtr, jint handleIndex) { + + auto* ctx = reinterpret_cast(contextPtr); + if (ctx && ctx->engine) { + ctx->engine->beginSelectionTransform(static_cast(handleIndex)); + } +} + +JNIEXPORT void JNICALL +Java_com_mathnotes_mobileink_MobileInkCanvasView_updateSelectionTransform( + JNIEnv* env, jobject obj, jlong contextPtr, jfloat x, jfloat y) { + + auto* ctx = reinterpret_cast(contextPtr); + if (ctx && ctx->engine) { + ctx->engine->updateSelectionTransform(x, y); + } +} + +JNIEXPORT void JNICALL +Java_com_mathnotes_mobileink_MobileInkCanvasView_finalizeSelectionTransform( + JNIEnv* env, jobject obj, jlong contextPtr) { + + auto* ctx = reinterpret_cast(contextPtr); + if (ctx && ctx->engine) { + ctx->engine->finalizeSelectionTransform(); + } +} + +JNIEXPORT void JNICALL +Java_com_mathnotes_mobileink_MobileInkCanvasView_cancelSelectionTransform( + JNIEnv* env, jobject obj, jlong contextPtr) { + + auto* ctx = reinterpret_cast(contextPtr); + if (ctx && ctx->engine) { + ctx->engine->cancelSelectionTransform(); + } +} + +JNIEXPORT jint JNICALL +Java_com_mathnotes_mobileink_MobileInkCanvasView_getSelectionCount(JNIEnv* env, jobject obj, jlong contextPtr) { + auto* ctx = reinterpret_cast(contextPtr); + if (ctx && ctx->engine) { + return ctx->engine->getSelectionCount(); + } + return 0; +} + +JNIEXPORT jfloatArray JNICALL +Java_com_mathnotes_mobileink_MobileInkCanvasView_getSelectionBounds(JNIEnv* env, jobject obj, jlong contextPtr) { + auto* ctx = reinterpret_cast(contextPtr); + if (ctx && ctx->engine) { + std::vector bounds = ctx->engine->getSelectionBounds(); + jfloatArray result = env->NewFloatArray(4); + env->SetFloatArrayRegion(result, 0, 4, bounds.data()); + return result; + } + return nullptr; +} + +// State queries +JNIEXPORT jboolean JNICALL +Java_com_mathnotes_mobileink_MobileInkCanvasView_canUndo(JNIEnv* env, jobject obj, jlong contextPtr) { + auto* ctx = reinterpret_cast(contextPtr); + if (ctx && ctx->engine) { + return ctx->engine->canUndo() ? JNI_TRUE : JNI_FALSE; + } + return JNI_FALSE; +} + +JNIEXPORT jboolean JNICALL +Java_com_mathnotes_mobileink_MobileInkCanvasView_canRedo(JNIEnv* env, jobject obj, jlong contextPtr) { + auto* ctx = reinterpret_cast(contextPtr); + if (ctx && ctx->engine) { + return ctx->engine->canRedo() ? JNI_TRUE : JNI_FALSE; + } + return JNI_FALSE; +} + +JNIEXPORT jboolean JNICALL +Java_com_mathnotes_mobileink_MobileInkCanvasView_isEmpty(JNIEnv* env, jobject obj, jlong contextPtr) { + auto* ctx = reinterpret_cast(contextPtr); + if (ctx && ctx->engine) { + return ctx->engine->isEmpty() ? JNI_TRUE : JNI_FALSE; + } + return JNI_TRUE; +} + +// Serialization +JNIEXPORT jbyteArray JNICALL +Java_com_mathnotes_mobileink_MobileInkCanvasView_serializeDrawing(JNIEnv* env, jobject obj, jlong contextPtr) { + auto* ctx = reinterpret_cast(contextPtr); + if (ctx && ctx->engine) { + std::vector data = ctx->engine->serializeDrawing(); + jbyteArray result = env->NewByteArray(data.size()); + env->SetByteArrayRegion(result, 0, data.size(), reinterpret_cast(data.data())); + return result; + } + LOGE("serializeDrawing: ctx or engine is null!"); + return nullptr; +} + +JNIEXPORT void JNICALL +Java_com_mathnotes_mobileink_MobileInkCanvasView_deserializeDrawing( + JNIEnv* env, jobject obj, jlong contextPtr, jbyteArray data) { + + auto* ctx = reinterpret_cast(contextPtr); + if (ctx && ctx->engine && data) { + jsize len = env->GetArrayLength(data); + jbyte* bytes = env->GetByteArrayElements(data, nullptr); + + std::vector vec(reinterpret_cast(bytes), + reinterpret_cast(bytes) + len); + ctx->engine->deserializeDrawing(vec); + + env->ReleaseByteArrayElements(data, bytes, JNI_ABORT); + } +} + +JNIEXPORT jboolean JNICALL +Java_com_mathnotes_mobileink_MobileInkCanvasView_renderGaneshToCurrentSurface( + JNIEnv* env, jobject obj, jlong contextPtr, jint width, jint height) { + + auto* ctx = reinterpret_cast(contextPtr); + if (!ctx || !ctx->engine || width <= 0 || height <= 0) { + LOGE("renderGaneshToCurrentSurface: invalid context or size"); + return JNI_FALSE; + } + + if (!ctx->ganeshContext) { + auto backendInterface = GrGLMakeNativeInterface(); + if (!backendInterface) { + LOGE("renderGaneshToCurrentSurface: failed to create GL interface"); + return JNI_FALSE; + } + ctx->ganeshContext = GrDirectContexts::MakeGL(backendInterface); + if (!ctx->ganeshContext) { + LOGE("renderGaneshToCurrentSurface: failed to create Ganesh context"); + return JNI_FALSE; + } + } + + GLint stencil = 0; + GLint samples = 0; + glGetIntegerv(GL_STENCIL_BITS, &stencil); + glGetIntegerv(GL_SAMPLES, &samples); + + const auto colorType = kRGBA_8888_SkColorType; + const auto maxSamples = + ctx->ganeshContext->maxSurfaceSampleCountForColorType(colorType); + if (samples > maxSamples) { + samples = maxSamples; + } + + glBindFramebuffer(GL_FRAMEBUFFER, 0); + glViewport(0, 0, width, height); + + if ( + !ctx->ganeshSurface || + ctx->ganeshSurfaceWidth != width || + ctx->ganeshSurfaceHeight != height || + ctx->ganeshSamples != samples || + ctx->ganeshStencil != stencil + ) { + ctx->ganeshSurface = nullptr; + + GrGLFramebufferInfo fbInfo; + fbInfo.fFBOID = 0; + fbInfo.fFormat = GR_GL_RGBA8; + + auto backendRenderTarget = + GrBackendRenderTargets::MakeGL(width, height, samples, stencil, fbInfo); + SkSurfaceProps surfaceProps(0, kRGB_H_SkPixelGeometry); + ctx->ganeshSurface = SkSurfaces::WrapBackendRenderTarget( + ctx->ganeshContext.get(), + backendRenderTarget, + kBottomLeft_GrSurfaceOrigin, + colorType, + nullptr, + &surfaceProps + ); + + if (!ctx->ganeshSurface) { + LOGE("renderGaneshToCurrentSurface: failed to wrap EGL render target"); + return JNI_FALSE; + } + + ctx->ganeshSurfaceWidth = width; + ctx->ganeshSurfaceHeight = height; + ctx->ganeshSamples = samples; + ctx->ganeshStencil = stencil; + } + + SkCanvas* canvas = ctx->ganeshSurface->getCanvas(); + renderWithDisplayColorOrdering(ctx->engine.get(), canvas); + ctx->ganeshContext->flushAndSubmit(ctx->ganeshSurface.get()); + return JNI_TRUE; +} + +JNIEXPORT void JNICALL +Java_com_mathnotes_mobileink_MobileInkCanvasView_releaseGaneshContext( + JNIEnv* env, jobject obj, jlong contextPtr) { + + auto* ctx = reinterpret_cast(contextPtr); + if (!ctx) { + return; + } + + ctx->ganeshSurface = nullptr; + ctx->ganeshSurfaceWidth = 0; + ctx->ganeshSurfaceHeight = 0; + ctx->ganeshSamples = -1; + ctx->ganeshStencil = -1; + + if (ctx->ganeshContext) { + ctx->ganeshContext->releaseResourcesAndAbandonContext(); + ctx->ganeshContext = nullptr; + } +} + +JNIEXPORT void JNICALL +Java_com_mathnotes_mobileink_MobileInkCanvasView_renderToPixelsScaled( + JNIEnv* env, jobject obj, jlong contextPtr, jobject bitmap, jfloat scale) { + + auto* ctx = reinterpret_cast(contextPtr); + if (!ctx || !ctx->engine) { + LOGE("renderToPixelsScaled: invalid context"); + return; + } + + AndroidBitmapInfo info; + if (AndroidBitmap_getInfo(env, bitmap, &info) != ANDROID_BITMAP_RESULT_SUCCESS) { + LOGE("renderToPixelsScaled: failed to get bitmap info"); + return; + } + + void* pixels = nullptr; + if (AndroidBitmap_lockPixels(env, bitmap, &pixels) != ANDROID_BITMAP_RESULT_SUCCESS) { + LOGE("renderToPixelsScaled: failed to lock pixels"); + return; + } + + SkImageInfo skInfo = SkImageInfo::MakeN32Premul(info.width, info.height); + sk_sp surface = SkSurfaces::WrapPixels(skInfo, pixels, info.stride); + if (!surface) { + AndroidBitmap_unlockPixels(env, bitmap); + LOGE("renderToPixelsScaled: failed to wrap bitmap pixels"); + return; + } + + SkCanvas* canvas = surface->getCanvas(); + canvas->clear(SK_ColorTRANSPARENT); + const float resolvedScale = scale > 0.0f ? scale : 1.0f; + canvas->save(); + canvas->scale(resolvedScale, resolvedScale); + renderWithDisplayColorOrdering(ctx->engine.get(), canvas); + canvas->restore(); + + AndroidBitmap_unlockPixels(env, bitmap); +} + +// Batch export multiple pages to PNG images +// This is a static method on MobileInkModule (not tied to a view instance) +// pagesDataArray: Array of byte arrays with serialized drawing data for each page +// backgroundTypes: Array of background type strings ("plain", "lined", "grid", "pdf") +// width, height: Canvas dimensions +// scale: Export scale factor +// Returns: Array of base64 PNG data URIs +JNIEXPORT jobjectArray JNICALL +Java_com_mathnotes_mobileink_MobileInkModule_nativeBatchExportPages( + JNIEnv* env, jclass clazz, + jobjectArray pagesDataArray, + jobjectArray backgroundTypesArray, + jobjectArray pdfBackgroundsArray, + jintArray pageIndicesArray, + jint width, jint height, jfloat scale) { + + int numPages = env->GetArrayLength(pagesDataArray); + if (numPages == 0) { + // Return empty array + jclass stringClass = env->FindClass("java/lang/String"); + return env->NewObjectArray(0, stringClass, nullptr); + } + + // Create temporary engine for batch processing + auto engine = std::make_unique(width, height); + + // Prepare vectors for batch export + std::vector> pagesData; + std::vector bgTypes; + std::vector> pdfBackgrounds; + pagesData.reserve(numPages); + bgTypes.reserve(numPages); + pdfBackgrounds.reserve(numPages); + + std::vector pageIndices(numPages); + if (pageIndicesArray != nullptr && env->GetArrayLength(pageIndicesArray) >= numPages) { + jint* rawPageIndices = env->GetIntArrayElements(pageIndicesArray, nullptr); + if (rawPageIndices != nullptr) { + for (int i = 0; i < numPages; i++) { + pageIndices[i] = rawPageIndices[i]; + } + env->ReleaseIntArrayElements(pageIndicesArray, rawPageIndices, JNI_ABORT); + } + } else { + for (int i = 0; i < numPages; i++) { + pageIndices[i] = i; + } + } + + for (int i = 0; i < numPages; i++) { + std::vector pageData; + auto pageBytes = (jbyteArray)env->GetObjectArrayElement(pagesDataArray, i); + if (pageBytes != nullptr) { + jsize len = env->GetArrayLength(pageBytes); + jbyte* bytes = env->GetByteArrayElements(pageBytes, nullptr); + if (bytes != nullptr) { + if (len > 0) { + pageData.assign( + reinterpret_cast(bytes), + reinterpret_cast(bytes) + len + ); + } + env->ReleaseByteArrayElements(pageBytes, bytes, JNI_ABORT); + } + env->DeleteLocalRef(pageBytes); + } + + pagesData.push_back(std::move(pageData)); + + // Get background type + jstring bgTypeStr = (jstring)env->GetObjectArrayElement(backgroundTypesArray, i); + const char* bgType = env->GetStringUTFChars(bgTypeStr, nullptr); + bgTypes.push_back(bgType ? bgType : "plain"); + env->ReleaseStringUTFChars(bgTypeStr, bgType); + env->DeleteLocalRef(bgTypeStr); + + sk_sp pdfImage = nullptr; + if (pdfBackgroundsArray != nullptr) { + jobject bitmap = env->GetObjectArrayElement(pdfBackgroundsArray, i); + if (bitmap != nullptr) { + AndroidBitmapInfo info; + if (AndroidBitmap_getInfo(env, bitmap, &info) == ANDROID_BITMAP_RESULT_SUCCESS) { + void* pixels = nullptr; + if (AndroidBitmap_lockPixels(env, bitmap, &pixels) == ANDROID_BITMAP_RESULT_SUCCESS) { + SkImageInfo skInfo = SkImageInfo::MakeN32Premul(info.width, info.height); + size_t dataSize = info.height * info.stride; + sk_sp data = SkData::MakeWithCopy(pixels, dataSize); + AndroidBitmap_unlockPixels(env, bitmap); + pdfImage = SkImages::RasterFromData(skInfo, data, info.stride); + } + } + env->DeleteLocalRef(bitmap); + } + } + pdfBackgrounds.push_back(pdfImage); + } + + + // Call batch export on the engine + std::vector results = engine->batchExportPages( + pagesData, bgTypes, pdfBackgrounds, pageIndices, scale); + + // Convert results to Java String array + jclass stringClass = env->FindClass("java/lang/String"); + jobjectArray resultArray = env->NewObjectArray(numPages, stringClass, nullptr); + + for (int i = 0; i < numPages && i < (int)results.size(); i++) { + if (!results[i].empty()) { + jstring resultStr = env->NewStringUTF(results[i].c_str()); + env->SetObjectArrayElement(resultArray, i, resultStr); + env->DeleteLocalRef(resultStr); + } else { + // Set empty string for failed exports + jstring emptyStr = env->NewStringUTF(""); + env->SetObjectArrayElement(resultArray, i, emptyStr); + env->DeleteLocalRef(emptyStr); + } + } + + return resultArray; +} + +JNIEXPORT jbyteArray JNICALL +Java_com_mathnotes_mobileink_MobileInkModule_nativeComposeContinuousWindow( + JNIEnv* env, jclass clazz, jobjectArray pageDataArray, jfloat pageHeight) { + + if (pageDataArray == nullptr || pageHeight <= 0.0f) { + return nullptr; + } + + int pageCount = env->GetArrayLength(pageDataArray); + if (pageCount <= 0) { + return nullptr; + } + + std::vector> pageData; + pageData.reserve(pageCount); + + for (int i = 0; i < pageCount; i++) { + std::vector data; + auto pageBytes = (jbyteArray)env->GetObjectArrayElement(pageDataArray, i); + if (pageBytes != nullptr) { + jsize len = env->GetArrayLength(pageBytes); + jbyte* bytes = env->GetByteArrayElements(pageBytes, nullptr); + if (bytes != nullptr && len > 0) { + data.assign( + reinterpret_cast(bytes), + reinterpret_cast(bytes) + len + ); + } + if (bytes != nullptr) { + env->ReleaseByteArrayElements(pageBytes, bytes, JNI_ABORT); + } + env->DeleteLocalRef(pageBytes); + } + pageData.push_back(std::move(data)); + } + + std::vector output = composeContinuousWindowBytes(pageData, pageHeight); + if (output.empty()) { + return nullptr; + } + + jbyteArray result = env->NewByteArray(output.size()); + env->SetByteArrayRegion( + result, + 0, + output.size(), + reinterpret_cast(output.data()) + ); + return result; +} + +JNIEXPORT jobjectArray JNICALL +Java_com_mathnotes_mobileink_MobileInkModule_nativeDecomposeContinuousWindow( + JNIEnv* env, jclass clazz, jbyteArray windowDataArray, jint pageCount, jfloat pageHeight) { + + jclass byteArrayClass = env->FindClass("[B"); + jobjectArray resultArray = env->NewObjectArray(std::max(0, static_cast(pageCount)), byteArrayClass, nullptr); + if (pageCount <= 0 || pageHeight <= 0.0f) { + return resultArray; + } + + std::vector windowData; + if (windowDataArray != nullptr) { + jsize len = env->GetArrayLength(windowDataArray); + jbyte* bytes = env->GetByteArrayElements(windowDataArray, nullptr); + if (bytes != nullptr && len > 0) { + windowData.assign( + reinterpret_cast(bytes), + reinterpret_cast(bytes) + len + ); + } + if (bytes != nullptr) { + env->ReleaseByteArrayElements(windowDataArray, bytes, JNI_ABORT); + } + } + + std::vector> pages = decomposeContinuousWindowBytes( + windowData.empty() ? nullptr : windowData.data(), + static_cast(windowData.size()), + pageCount, + pageHeight + ); + + if (pages.empty() && pageCount > 0 && !windowData.empty()) { + return resultArray; + } + + for (int i = 0; i < pageCount && i < static_cast(pages.size()); i++) { + const auto& page = pages[i]; + if (page.empty()) { + continue; + } + jbyteArray pageArray = env->NewByteArray(page.size()); + env->SetByteArrayRegion( + pageArray, + 0, + page.size(), + reinterpret_cast(page.data()) + ); + env->SetObjectArrayElement(resultArray, i, pageArray); + env->DeleteLocalRef(pageArray); + } + + return resultArray; +} + +} // extern "C" diff --git a/android/src/main/java/com/mathnotes/mobileink/MobileInkCanvasView.kt b/android/src/main/java/com/mathnotes/mobileink/MobileInkCanvasView.kt new file mode 100644 index 0000000..5b14be1 --- /dev/null +++ b/android/src/main/java/com/mathnotes/mobileink/MobileInkCanvasView.kt @@ -0,0 +1,1333 @@ +package com.mathnotes.mobileink + +import android.content.Context +import android.graphics.Bitmap +import android.graphics.SurfaceTexture +import android.opengl.EGL14 +import android.opengl.EGLConfig +import android.opengl.EGLContext +import android.opengl.EGLDisplay +import android.opengl.EGLSurface +import android.os.Handler +import android.os.HandlerThread +import android.os.Looper +import android.os.SystemClock +import android.view.MotionEvent +import android.view.Surface +import android.view.TextureView +import com.facebook.react.bridge.Arguments +import com.facebook.react.bridge.ReactContext +import com.facebook.react.uimanager.events.RCTEventEmitter + +class MobileInkCanvasView(context: Context) : TextureView(context), TextureView.SurfaceTextureListener { + + private val perpendicularAltitude: Float = Math.PI.toFloat() / 2f + private val minimumStylusPressure: Float = 0.015f + private val stylusPressureExponent: Float = 0.88f + + @Volatile private var drawingEngine: Long = 0 + @Volatile private var viewWidth: Int = 0 + @Volatile private var viewHeight: Int = 0 + private var engineWidth: Int = 0 + private var engineHeight: Int = 0 + private var nativeLibraryAvailable: Boolean = false + @Volatile private var surfaceReady: Boolean = false + private val renderThreadLock = Any() + private var renderThread: HandlerThread? = null + private var renderHandler: Handler? = null + private var renderSurface: Surface? = null + private var eglDisplay: EGLDisplay = EGL14.EGL_NO_DISPLAY + private var eglContext: EGLContext = EGL14.EGL_NO_CONTEXT + private var eglSurface: EGLSurface = EGL14.EGL_NO_SURFACE + + // Tool state tracking + private var currentToolType: String = "pen" + private var currentEraserMode: String = "pixel" + private var currentStrokeWidth: Float = 3f + private var currentStrokeColor: Int = android.graphics.Color.BLACK + private val holdToShapeDelayMs: Long = 300 + private val holdToShapeHandler = Handler(Looper.getMainLooper()) + private val holdToShapeRunnable = Runnable { showHoldToShapePreview() } + + // Selection move state + private var isMovingSelection: Boolean = false + private var isTransformingSelection: Boolean = false + private var hasSelectionMoveDelta: Boolean = false + private var selectionTransformHandleIndex: Int = -1 + private var lastDragX: Float = 0f + private var lastDragY: Float = 0f + + // Eraser cursor state (for pixel eraser) + private var eraserCursorX: Float = 0f + private var eraserCursorY: Float = 0f + private var showEraserCursor: Boolean = false + + // Background type for pattern rendering (handled in C++ Skia layer) + private var currentBackgroundType: String = "plain" + private var currentPdfBackgroundUri: String? = null + + // Drawing policy: "default", "anyinput", or "pencilonly" + // When "pencilonly", only stylus touches are processed for drawing + var drawingPolicy: String = "default" + var renderBackend: String = "ganesh" + var renderSuspended: Boolean = false + set(value) { + if (field == value) return + field = value + if (!value) { + requestInkRender(force = true) + } + } + + init { + // Try to load native library - don't crash if it fails + nativeLibraryAvailable = ensureLibraryLoaded() + ensureRenderHandler() + isOpaque = false + isClickable = true + isFocusable = true + surfaceTextureListener = this + } + + override fun onSurfaceTextureAvailable(surfaceTexture: SurfaceTexture, width: Int, height: Int) { + configureAvailableTexture(surfaceTexture, width, height) + } + + override fun onSurfaceTextureSizeChanged(surfaceTexture: SurfaceTexture, width: Int, height: Int) { + configureAvailableTexture(surfaceTexture, width, height) + } + + override fun onSurfaceTextureDestroyed(surfaceTexture: SurfaceTexture): Boolean { + runOnGlThreadSync(2000) { + destroyRenderSurface() + true + } + return true + } + + override fun onSurfaceTextureUpdated(surfaceTexture: SurfaceTexture) = Unit + + override fun onSizeChanged(width: Int, height: Int, oldWidth: Int, oldHeight: Int) { + super.onSizeChanged(width, height, oldWidth, oldHeight) + val texture = surfaceTexture + if (isAvailable && texture != null) { + configureAvailableTexture(texture, width, height) + } + } + + private fun configureAvailableTexture(surfaceTexture: SurfaceTexture, width: Int, height: Int) { + if (width <= 0 || height <= 0) { + return + } + + surfaceTexture.setDefaultBufferSize(width, height) + queueEvent { + if ( + !surfaceReady || + eglDisplay == EGL14.EGL_NO_DISPLAY || + eglContext == EGL14.EGL_NO_CONTEXT || + eglSurface == EGL14.EGL_NO_SURFACE + ) { + createRenderSurface(surfaceTexture, width, height) + } else if (makeRenderContextCurrent()) { + configureSurfaceSize(width, height) + renderFrame() + } + } + } + + private fun configureSurfaceSize(width: Int, height: Int) { + viewWidth = width + viewHeight = height + + if (width <= 0 || height <= 0) { + return + } + if (!nativeLibraryAvailable) { + android.util.Log.e("MobileInkCanvasView", "Cannot configure drawing engine because native library is unavailable") + return + } + + val shouldPreserveDrawing = drawingEngine != 0L + if (drawingEngine == 0L || engineWidth != width || engineHeight != height) { + configureDrawingEngine(width, height, shouldPreserveDrawing) + } + + // Re-apply PDF background if we have one (needs to be re-rendered at new size) + if (!currentPdfBackgroundUri.isNullOrEmpty()) { + setPdfBackgroundUri(currentPdfBackgroundUri) + } + } + + private fun createRenderSurface(surfaceTexture: SurfaceTexture, width: Int, height: Int) { + if (width <= 0 || height <= 0) { + return + } + + destroyRenderSurface() + + renderSurface = Surface(surfaceTexture) + eglDisplay = EGL14.eglGetDisplay(EGL14.EGL_DEFAULT_DISPLAY) + if (eglDisplay == EGL14.EGL_NO_DISPLAY) { + android.util.Log.e("MobileInkCanvasView", "Unable to get EGL display") + return + } + + val version = IntArray(2) + if (!EGL14.eglInitialize(eglDisplay, version, 0, version, 1)) { + android.util.Log.e("MobileInkCanvasView", "Unable to initialize EGL") + destroyRenderSurface() + return + } + + val configAttributes = intArrayOf( + EGL14.EGL_RENDERABLE_TYPE, EGL14.EGL_OPENGL_ES2_BIT, + EGL14.EGL_SURFACE_TYPE, EGL14.EGL_WINDOW_BIT, + EGL14.EGL_RED_SIZE, 8, + EGL14.EGL_GREEN_SIZE, 8, + EGL14.EGL_BLUE_SIZE, 8, + EGL14.EGL_ALPHA_SIZE, 8, + EGL14.EGL_DEPTH_SIZE, 0, + EGL14.EGL_STENCIL_SIZE, 0, + EGL14.EGL_NONE, + ) + val configs = arrayOfNulls(1) + val configCount = IntArray(1) + if (!EGL14.eglChooseConfig( + eglDisplay, + configAttributes, + 0, + configs, + 0, + configs.size, + configCount, + 0, + ) || configCount[0] == 0 || configs[0] == null + ) { + android.util.Log.e("MobileInkCanvasView", "Unable to choose EGL config") + destroyRenderSurface() + return + } + + val resolvedConfig = configs[0] + if (resolvedConfig == null) { + android.util.Log.e("MobileInkCanvasView", "EGL config was unexpectedly null") + destroyRenderSurface() + return + } + + eglContext = EGL14.eglCreateContext( + eglDisplay, + resolvedConfig, + EGL14.EGL_NO_CONTEXT, + intArrayOf(EGL14.EGL_CONTEXT_CLIENT_VERSION, 2, EGL14.EGL_NONE), + 0, + ) + if (eglContext == EGL14.EGL_NO_CONTEXT) { + android.util.Log.e("MobileInkCanvasView", "Unable to create EGL context") + destroyRenderSurface() + return + } + + eglSurface = EGL14.eglCreateWindowSurface( + eglDisplay, + resolvedConfig, + renderSurface, + intArrayOf(EGL14.EGL_NONE), + 0, + ) + if (eglSurface == EGL14.EGL_NO_SURFACE) { + android.util.Log.e("MobileInkCanvasView", "Unable to create EGL window surface") + destroyRenderSurface() + return + } + + if (!makeRenderContextCurrent()) { + destroyRenderSurface() + return + } + + surfaceReady = true + configureSurfaceSize(width, height) + renderFrame() + } + + private fun makeRenderContextCurrent(): Boolean { + if ( + eglDisplay == EGL14.EGL_NO_DISPLAY || + eglContext == EGL14.EGL_NO_CONTEXT || + eglSurface == EGL14.EGL_NO_SURFACE + ) { + return false + } + + val success = EGL14.eglMakeCurrent(eglDisplay, eglSurface, eglSurface, eglContext) + if (!success) { + android.util.Log.e("MobileInkCanvasView", "Unable to make EGL context current") + } + return success + } + + private fun destroyRenderSurface() { + surfaceReady = false + + if (drawingEngine != 0L && eglDisplay != EGL14.EGL_NO_DISPLAY) { + makeRenderContextCurrent() + releaseGaneshContext(drawingEngine) + } + + if (eglDisplay != EGL14.EGL_NO_DISPLAY) { + EGL14.eglMakeCurrent( + eglDisplay, + EGL14.EGL_NO_SURFACE, + EGL14.EGL_NO_SURFACE, + EGL14.EGL_NO_CONTEXT, + ) + if (eglSurface != EGL14.EGL_NO_SURFACE) { + EGL14.eglDestroySurface(eglDisplay, eglSurface) + } + if (eglContext != EGL14.EGL_NO_CONTEXT) { + EGL14.eglDestroyContext(eglDisplay, eglContext) + } + EGL14.eglTerminate(eglDisplay) + } + + eglSurface = EGL14.EGL_NO_SURFACE + eglContext = EGL14.EGL_NO_CONTEXT + eglDisplay = EGL14.EGL_NO_DISPLAY + renderSurface?.release() + renderSurface = null + } + + private fun configureDrawingEngine(width: Int, height: Int, preserveExistingDrawing: Boolean) { + val preservedDrawing = if (preserveExistingDrawing && drawingEngine != 0L) { + try { + serializeDrawing(drawingEngine) + } catch (e: Exception) { + android.util.Log.e("MobileInkCanvasView", "Failed to preserve drawing while resizing", e) + null + } + } else { + null + } + + if (drawingEngine != 0L) { + releaseGaneshContext(drawingEngine) + destroyDrawingEngine(drawingEngine) + drawingEngine = 0 + } + + drawingEngine = createDrawingEngine(width, height) + engineWidth = width + engineHeight = height + + if (drawingEngine == 0L) { + return + } + + setBackgroundType(drawingEngine, currentBackgroundType) + applyCurrentToolToEngine(drawingEngine) + + if (preservedDrawing != null && preservedDrawing.isNotEmpty()) { + deserializeDrawing(drawingEngine, preservedDrawing) + } + + resetTransientInteractionState() + post { emitSelectionChange() } + } + + private fun renderFrame() { + val engine = drawingEngine + if (!surfaceReady || engine == 0L || viewWidth <= 0 || viewHeight <= 0) { + return + } + if (!makeRenderContextCurrent()) { + return + } + if (renderGaneshToCurrentSurface(engine, viewWidth, viewHeight)) { + EGL14.eglSwapBuffers(eglDisplay, eglSurface) + } + } + + private fun ensureRenderHandler(): Handler { + synchronized(renderThreadLock) { + val existingThread = renderThread + val existingHandler = renderHandler + if (existingThread != null && existingThread.isAlive && existingHandler != null) { + return existingHandler + } + + val thread = HandlerThread("MobileInkCanvasViewRender").also { it.start() } + val handler = Handler(thread.looper) + renderThread = thread + renderHandler = handler + return handler + } + } + + private fun stopRenderThread() { + val threadToStop = synchronized(renderThreadLock) { + val thread = renderThread + renderThread = null + renderHandler = null + thread + } + threadToStop?.quitSafely() + } + + private fun queueEvent(block: () -> Unit) { + val handler = ensureRenderHandler() + if (Looper.myLooper() == handler.looper) { + block() + return + } + + if (!handler.post(block)) { + android.util.Log.e("MobileInkCanvasView", "Failed to post GL operation to render thread") + } + } + + /** + * Set background type for pattern rendering. + * Patterns are rendered in the C++ Skia layer. + */ + fun setBackgroundType(type: String) { + currentBackgroundType = type + queueEvent { + if (drawingEngine != 0L) { + setBackgroundType(drawingEngine, type) + } + } + requestInkRender() + } + + /** + * Set PDF background URI for PDF page rendering. + * Loads the PDF, renders it to a bitmap, and passes to C++ for display. + */ + fun setPdfBackgroundUri(uri: String?) { + currentPdfBackgroundUri = uri + + if (uri.isNullOrEmpty()) { + // Clear PDF background + queueEvent { + if (drawingEngine != 0L) { + setPdfBackgroundBitmap(drawingEngine, null) + } + } + requestInkRender() + return + } + + // Load PDF and render to bitmap on background thread + Thread { + val pdfBitmap = PdfLoader.loadAndRenderPdf(context, uri, viewWidth, viewHeight) + if (pdfBitmap != null) { + queueEvent { + if (drawingEngine != 0L) { + setPdfBackgroundBitmap(drawingEngine, pdfBitmap) + setBackgroundType(drawingEngine, "pdf") + } + // Recycle bitmap after passing to C++ (it copies the pixels) + pdfBitmap.recycle() + } + requestInkRender() + } else { + android.util.Log.e("MobileInkCanvasView", "Failed to load PDF background") + } + }.start() + } + + // Track the registered React tag to properly unregister on detach + private var registeredTag: Int = -1 + + override fun onAttachedToWindow() { + super.onAttachedToWindow() + ensureRenderHandler() + // Register this view with MobileInkModule for bridgeless architecture support + // The 'id' is set by React Native to the view's React tag + if (id != -1 && id != 0) { + registeredTag = id + MobileInkModule.registerView(id, this) + } + val texture = surfaceTexture + val currentWidth = width + val currentHeight = height + if (!surfaceReady && isAvailable && texture != null && currentWidth > 0 && currentHeight > 0) { + configureAvailableTexture(texture, currentWidth, currentHeight) + } + } + + // Called by React Native when the view's id (React tag) is set + override fun setId(id: Int) { + super.setId(id) + // Register with the new id if we haven't already and we're attached + if (id != -1 && id != 0 && isAttachedToWindow && registeredTag != id) { + // Unregister old tag if any + if (registeredTag != -1) { + MobileInkModule.unregisterView(registeredTag) + } + registeredTag = id + MobileInkModule.registerView(id, this) + } + } + + override fun onDetachedFromWindow() { + cancelHoldToShapePreview() + + // Unregister this view from MobileInkModule + if (registeredTag != -1) { + MobileInkModule.unregisterView(registeredTag) + registeredTag = -1 + } + + runOnGlThreadSync(2000) { + if (drawingEngine != 0L) { + if (eglDisplay != EGL14.EGL_NO_DISPLAY) { + makeRenderContextCurrent() + releaseGaneshContext(drawingEngine) + } + destroyDrawingEngine(drawingEngine) + drawingEngine = 0 + } + destroyRenderSurface() + true + } + stopRenderThread() + super.onDetachedFromWindow() + } + + private fun requestInkRender(force: Boolean = false) { + if (!renderSuspended || force) { + queueEvent { + renderFrame() + } + } + } + + private fun canPreviewHoldToShape(): Boolean { + return currentToolType == "pen" || + currentToolType == "pencil" || + currentToolType == "marker" || + currentToolType == "highlighter" || + currentToolType == "crayon" || + currentToolType == "calligraphy" + } + + private fun scheduleHoldToShapePreview() { + cancelHoldToShapePreview() + if (!canPreviewHoldToShape()) return + holdToShapeHandler.postDelayed(holdToShapeRunnable, holdToShapeDelayMs) + } + + private fun cancelHoldToShapePreview() { + holdToShapeHandler.removeCallbacks(holdToShapeRunnable) + } + + private fun showHoldToShapePreview() { + if (!canPreviewHoldToShape() || drawingEngine == 0L) return + + val timestamp = SystemClock.uptimeMillis() + queueEvent { + if (drawingEngine != 0L) { + updateHoldShapePreview(drawingEngine, timestamp) + } + } + requestInkRender() + } + + private fun normalizedPressure(rawPressure: Float, isStylusInput: Boolean): Float { + val clamped = if (rawPressure.isFinite()) rawPressure.coerceIn(0f, 1f) else 1f + return if (isStylusInput) { + maxOf(minimumStylusPressure, Math.pow(clamped.toDouble(), stylusPressureExponent.toDouble()).toFloat()) + } else { + maxOf(0.1f, clamped) + } + } + + private fun normalizedAltitudeFromTilt(rawTilt: Float, isStylusInput: Boolean): Float { + if (!isStylusInput) { + return perpendicularAltitude + } + val altitude = perpendicularAltitude - rawTilt + return if (altitude.isFinite()) altitude.coerceIn(0f, perpendicularAltitude) else perpendicularAltitude + } + + private fun selectionBoundsContain(x: Float, y: Float, bounds: FloatArray, padding: Float): Boolean { + return x >= bounds[0] - padding && + x <= bounds[2] + padding && + y >= bounds[1] - padding && + y <= bounds[3] + padding + } + + private fun selectionHandleHitTest(x: Float, y: Float, bounds: FloatArray): Int? { + if (bounds.size != 4 || bounds[2] <= bounds[0] || bounds[3] <= bounds[1]) { + return null + } + + val centerX = (bounds[0] + bounds[2]) * 0.5f + val centerY = (bounds[1] + bounds[3]) * 0.5f + val handles = arrayOf( + 0 to floatArrayOf(bounds[0], bounds[1]), + 1 to floatArrayOf(centerX, bounds[1]), + 2 to floatArrayOf(bounds[2], bounds[1]), + 3 to floatArrayOf(bounds[0], centerY), + 4 to floatArrayOf(bounds[2], centerY), + 5 to floatArrayOf(bounds[0], bounds[3]), + 6 to floatArrayOf(centerX, bounds[3]), + 7 to floatArrayOf(bounds[2], bounds[3]) + ) + val hitRadius = 28f * resources.displayMetrics.density + val hitRadiusSquared = hitRadius * hitRadius + + for ((handleIndex, point) in handles) { + val dx = x - point[0] + val dy = y - point[1] + if (dx * dx + dy * dy <= hitRadiusSquared) { + return handleIndex + } + } + return null + } + + private fun handleFingerSelectionTouch(x: Float, y: Float): Boolean { + if (currentToolType == "text" || drawingEngine == 0L) { + return false + } + + cancelHoldToShapePreview() + showEraserCursor = false + + val selectionCount = getSelectionCount() + val bounds = if (selectionCount > 0) getSelectionBounds() else null + if (bounds != null) { + val handleIndex = selectionHandleHitTest(x, y, bounds) + if (handleIndex != null) { + queueEvent { + if (drawingEngine != 0L) { + beginSelectionTransform(drawingEngine, handleIndex) + } + } + isTransformingSelection = true + selectionTransformHandleIndex = handleIndex + hasSelectionMoveDelta = false + lastDragX = x + lastDragY = y + requestInkRender() + return true + } + + if (selectionBoundsContain(x, y, bounds, 24f * resources.displayMetrics.density)) { + isMovingSelection = true + hasSelectionMoveDelta = false + lastDragX = x + lastDragY = y + requestInkRender() + return true + } + } + + val hadSelection = selectionCount > 0 + val selectedShape = runOnGlThreadSync { + if (drawingEngine != 0L) { + if (hadSelection) { + clearSelection(drawingEngine) + } + selectShapeStrokeAt(drawingEngine, x, y) + } else { + false + } + } ?: false + + if (selectedShape) { + isMovingSelection = true + hasSelectionMoveDelta = false + lastDragX = x + lastDragY = y + requestInkRender() + emitSelectionChange() + return true + } + + if (hadSelection) { + requestInkRender() + emitSelectionChange() + return true + } + + return false + } + + override fun onTouchEvent(event: MotionEvent): Boolean { + if (!nativeLibraryAvailable) { + android.util.Log.e("MobileInkCanvasView", "onTouchEvent: native library not available") + return false + } + if (drawingEngine == 0L) { + android.util.Log.e("MobileInkCanvasView", "onTouchEvent: drawingEngine is 0") + return false + } + + val x = event.x + val y = event.y + val toolType = event.getToolType(0) + val isStylusInput = toolType == MotionEvent.TOOL_TYPE_STYLUS || + toolType == MotionEvent.TOOL_TYPE_ERASER + + if (event.actionMasked == MotionEvent.ACTION_DOWN && + !isStylusInput && + handleFingerSelectionTouch(x, y) + ) { + parent?.requestDisallowInterceptTouchEvent(true) + return true + } + + val isSelectionInteraction = currentToolType == "select" || isMovingSelection || isTransformingSelection + if (drawingPolicy == "pencilonly" && !isStylusInput && !isSelectionInteraction) { + return false + } + + // CRITICAL: Request parent to not intercept touch events + // This is essential for drawing - without it, ScrollView or other parents + // will intercept ACTION_MOVE events, causing only dots to appear + parent?.requestDisallowInterceptTouchEvent(true) + + val pressure = normalizedPressure(event.pressure, isStylusInput) + + // Get stylus tilt (altitude) - AXIS_TILT is tilt angle from perpendicular + // 0 = perpendicular to screen, pi/2 = parallel to screen + val tilt = event.getAxisValue(MotionEvent.AXIS_TILT) + val altitude = normalizedAltitudeFromTilt(tilt, isStylusInput) + + // Get stylus orientation (azimuth) - angle around the perpendicular axis + val azimuth = if (isStylusInput) event.getAxisValue(MotionEvent.AXIS_ORIENTATION) else 0f + val eventTimestamp = event.eventTime + + when (event.actionMasked) { + MotionEvent.ACTION_DOWN -> { + // For select tool, check if tapping inside existing selection to move it + if (currentToolType == "select" && drawingEngine != 0L) { + val selectionCount = getSelectionCount() + if (selectionCount > 0) { + val bounds = getSelectionBounds() + if (bounds != null && isPointInBounds(x, y, bounds)) { + // Start moving the selection + isMovingSelection = true + hasSelectionMoveDelta = false + lastDragX = x + lastDragY = y + cancelHoldToShapePreview() + return true + } + } + } + + // Update eraser cursor position for pixel eraser + if (currentToolType == "eraser" && currentEraserMode == "pixel") { + eraserCursorX = x + eraserCursorY = y + showEraserCursor = true + } + + // Queue touch event to GL thread to avoid race conditions with render thread + queueEvent { + if (drawingEngine != 0L) { + // Set eraser cursor in C++ engine (radius is half the stroke width) + if (currentToolType == "eraser" && currentEraserMode == "pixel") { + setEraserCursor(drawingEngine, x, y, currentStrokeWidth / 2f, true) + } + touchBegan(drawingEngine, x, y, pressure, azimuth, altitude, eventTimestamp, isStylusInput) + } + } + requestInkRender() + scheduleHoldToShapePreview() + sendDrawingBeginEvent(x, y) + } + MotionEvent.ACTION_MOVE -> { + if (isTransformingSelection && drawingEngine != 0L) { + cancelHoldToShapePreview() + val moved = x != lastDragX || y != lastDragY + if (moved) { + hasSelectionMoveDelta = true + } + queueEvent { + if (drawingEngine != 0L) { + updateSelectionTransform(drawingEngine, x, y) + post { emitSelectionChange() } + } + } + lastDragX = x + lastDragY = y + requestInkRender() + return true + } + + // Handle selection move + if (isMovingSelection && drawingEngine != 0L) { + cancelHoldToShapePreview() + val dx = x - lastDragX + val dy = y - lastDragY + if (dx != 0f || dy != 0f) { + hasSelectionMoveDelta = true + queueEvent { + if (drawingEngine != 0L) { + moveSelection(drawingEngine, dx, dy) + post { emitSelectionChange() } + } + } + } + lastDragX = x + lastDragY = y + requestInkRender() + return true + } + + // Update eraser cursor position (local state) + if (currentToolType == "eraser" && currentEraserMode == "pixel") { + eraserCursorX = x + eraserCursorY = y + } + + // Pixel eraser interpolates between samples in C++ and scans + // existing strokes per sample, so replaying Android history here + // multiplies the expensive work without improving coverage. + val shouldProcessHistoricalPoints = + currentToolType != "eraser" || currentEraserMode != "pixel" + + // Collect historical points for batch processing + val historySize = event.historySize + val historicalPoints = mutableListOf() + val historicalTimestamps = mutableListOf() + if (shouldProcessHistoricalPoints) { + for (i in 0 until historySize) { + val hx = event.getHistoricalX(i) + val hy = event.getHistoricalY(i) + val hp = normalizedPressure(event.getHistoricalPressure(i), isStylusInput) + val hTilt = event.getHistoricalAxisValue(MotionEvent.AXIS_TILT, i) + val hAltitude = normalizedAltitudeFromTilt(hTilt, isStylusInput) + val hAzimuth = if (isStylusInput) { + event.getHistoricalAxisValue(MotionEvent.AXIS_ORIENTATION, i) + } else { + 0f + } + historicalPoints.add(floatArrayOf(hx, hy, hp, hAzimuth, hAltitude)) + historicalTimestamps.add(event.getHistoricalEventTime(i)) + } + } + + // Queue all touch moves to GL thread + queueEvent { + if (drawingEngine != 0L) { + // Update eraser cursor in C++ engine + if (currentToolType == "eraser" && currentEraserMode == "pixel") { + setEraserCursor(drawingEngine, x, y, currentStrokeWidth / 2f, true) + } + // Process historical points + for ((index, point) in historicalPoints.withIndex()) { + touchMoved( + drawingEngine, + point[0], + point[1], + point[2], + point[3], + point[4], + historicalTimestamps[index], + isStylusInput + ) + } + // Process current point + touchMoved(drawingEngine, x, y, pressure, azimuth, altitude, eventTimestamp, isStylusInput) + } + } + requestInkRender() + scheduleHoldToShapePreview() + } + MotionEvent.ACTION_UP, MotionEvent.ACTION_CANCEL -> { + cancelHoldToShapePreview() + val isCancel = event.actionMasked == MotionEvent.ACTION_CANCEL + + if (isTransformingSelection && drawingEngine != 0L) { + val didTransformSelection = hasSelectionMoveDelta && !isCancel + queueEvent { + if (drawingEngine != 0L) { + if (didTransformSelection) { + finalizeSelectionTransform(drawingEngine) + } else { + cancelSelectionTransform(drawingEngine) + } + post { emitSelectionChange() } + } + } + isTransformingSelection = false + selectionTransformHandleIndex = -1 + hasSelectionMoveDelta = false + requestInkRender() + if (didTransformSelection) { + sendEvent("onDrawingChange", Arguments.createMap()) + } + return true + } + + // Finalize selection move + if (isMovingSelection && drawingEngine != 0L) { + val didMoveSelection = hasSelectionMoveDelta && !isCancel + if (didMoveSelection) { + queueEvent { + if (drawingEngine != 0L) { + finalizeMove(drawingEngine) + post { emitSelectionChange() } + } + } + } else { + post { emitSelectionChange() } + } + isMovingSelection = false + hasSelectionMoveDelta = false + requestInkRender() + if (didMoveSelection) { + sendEvent("onDrawingChange", Arguments.createMap()) + } + return true + } + + // Hide eraser cursor (local state) + showEraserCursor = false + + // Queue touch end to GL thread to avoid race conditions + queueEvent { + if (drawingEngine != 0L) { + setEraserCursor(drawingEngine, 0f, 0f, 0f, false) + touchEnded(drawingEngine, if (isCancel) 0L else eventTimestamp) + } + if (currentToolType == "select") { + post { emitSelectionChange() } + } + } + requestInkRender() + if (currentToolType != "select") { + sendEvent("onDrawingChange", Arguments.createMap()) + } + } + } + + return true + } + + fun clear() { + clearCanvasForLoad() + sendEvent("onDrawingChange", Arguments.createMap()) + } + + fun clearCanvasForLoad(): Boolean { + val success = runOnGlThreadSync { + if (drawingEngine != 0L) { + clearCanvas(drawingEngine) + true + } else { + false + } + } ?: false + + if (success) { + resetTransientInteractionState() + requestInkRender() + emitSelectionChange() + } + + return success + } + + fun clearCanvasAsyncForLoad() { + queueEvent { + if (drawingEngine != 0L) { + clearCanvas(drawingEngine) + post { emitSelectionChange() } + } + } + requestInkRender() + } + + fun undo() { + queueEvent { + if (drawingEngine != 0L) { + undoStroke(drawingEngine) + post { emitSelectionChange() } + } + } + requestInkRender() + sendEvent("onDrawingChange", Arguments.createMap()) + } + + fun redo() { + queueEvent { + if (drawingEngine != 0L) { + redoStroke(drawingEngine) + post { emitSelectionChange() } + } + } + requestInkRender() + sendEvent("onDrawingChange", Arguments.createMap()) + } + + fun setTool(toolType: String, width: Float, color: Int) { + cancelHoldToShapePreview() + currentToolType = toolType + currentEraserMode = "pixel" + currentStrokeWidth = width + currentStrokeColor = color + resetTransientInteractionState() + + queueEvent { + if (drawingEngine != 0L) { + if (toolType != "select") { + clearSelection(drawingEngine) + post { emitSelectionChange() } + } + applyCurrentToolToEngine(drawingEngine) + } + } + requestInkRender() + } + + fun setToolWithParams(toolType: String, width: Float, color: Int, eraserMode: String?) { + cancelHoldToShapePreview() + + // Update tool state (local - doesn't need queuing) + currentToolType = toolType + currentEraserMode = eraserMode ?: "pixel" + currentStrokeWidth = width + currentStrokeColor = color + resetTransientInteractionState() + + // Hide eraser cursor if switching away from pixel eraser + if (toolType != "eraser" || eraserMode != "pixel") { + showEraserCursor = false + } + + // Queue engine operations to GL thread + queueEvent { + if (drawingEngine != 0L) { + // Clear selection when switching away from select tool + if (toolType != "select") { + clearSelection(drawingEngine) + post { emitSelectionChange() } + } + applyCurrentToolToEngine(drawingEngine) + } + } + requestInkRender() + } + + private fun applyCurrentToolToEngine(engine: Long) { + setToolWithParams( + engine, + currentToolType, + currentStrokeWidth, + currentStrokeColor, + currentEraserMode + ) + } + + private fun resetTransientInteractionState() { + isMovingSelection = false + isTransformingSelection = false + hasSelectionMoveDelta = false + selectionTransformHandleIndex = -1 + lastDragX = 0f + lastDragY = 0f + } + + // Helper to check if a point is inside selection bounds + private fun isPointInBounds(x: Float, y: Float, bounds: FloatArray): Boolean { + // bounds is [minX, minY, maxX, maxY] + return x >= bounds[0] && x <= bounds[2] && y >= bounds[1] && y <= bounds[3] + } + + fun setDrawingBackgroundColor(color: Int) { + setBackgroundColor(color) + requestInkRender() + } + + // Selection operations + fun selectAt(x: Float, y: Float): Boolean { + val result = runOnGlThreadSync { + if (drawingEngine != 0L) selectStrokeAt(drawingEngine, x, y) else false + } ?: false + if (result) { + requestInkRender() + emitSelectionChange() + } + return result + } + + fun clearSelection() { + queueEvent { + if (drawingEngine != 0L) { + clearSelection(drawingEngine) + post { emitSelectionChange() } + } + } + requestInkRender() + } + + fun deleteSelection() { + queueEvent { + if (drawingEngine != 0L) { + deleteSelection(drawingEngine) + post { emitSelectionChange() } + } + } + requestInkRender() + sendEvent("onDrawingChange", Arguments.createMap()) + } + + fun copySelection() { + queueEvent { + if (drawingEngine != 0L) { + copySelection(drawingEngine) + } + } + } + + fun pasteSelection(offsetX: Float, offsetY: Float) { + queueEvent { + if (drawingEngine != 0L) { + pasteSelection(drawingEngine, offsetX, offsetY) + post { emitSelectionChange() } + } + } + requestInkRender() + sendEvent("onDrawingChange", Arguments.createMap()) + } + + fun moveSelection(dx: Float, dy: Float) { + queueEvent { + if (drawingEngine != 0L) { + moveSelection(drawingEngine, dx, dy) + post { emitSelectionChange() } + } + } + requestInkRender() + } + + fun finalizeMoveSelection() { + queueEvent { + if (drawingEngine != 0L) { + finalizeMove(drawingEngine) + post { emitSelectionChange() } + } + } + } + + fun getSelectionCount(): Int { + return runOnGlThreadSync { + if (drawingEngine != 0L) getSelectionCount(drawingEngine) else 0 + } ?: 0 + } + + fun getSelectionBounds(): FloatArray? { + return runOnGlThreadSync { + if (drawingEngine != 0L) getSelectionBounds(drawingEngine) else null + } + } + + // State queries + fun canUndo(): Boolean = runOnGlThreadSync { + drawingEngine != 0L && canUndo(drawingEngine) + } ?: false + fun canRedo(): Boolean = runOnGlThreadSync { + drawingEngine != 0L && canRedo(drawingEngine) + } ?: false + fun isEmpty(): Boolean = runOnGlThreadSync { + drawingEngine == 0L || isEmpty(drawingEngine) + } ?: true + + // Helper for synchronous GL thread operations with timeout + private fun runOnGlThreadSync(timeoutMs: Long = 2000, block: () -> T?): T? { + val handler = ensureRenderHandler() + if (Looper.myLooper() == handler.looper) { + return block() + } + + val latch = java.util.concurrent.CountDownLatch(1) + var result: T? = null + var failure: Throwable? = null + if (!handler.post { + try { + result = block() + } catch (throwable: Throwable) { + failure = throwable + } finally { + latch.countDown() + } + }) { + android.util.Log.e("MobileInkCanvasView", "Failed to post synchronous GL operation") + return null + } + try { + val completed = latch.await(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS) + if (!completed) { + android.util.Log.e("MobileInkCanvasView", "GL thread operation timed out after ${timeoutMs}ms") + } + } catch (e: InterruptedException) { + android.util.Log.e("MobileInkCanvasView", "GL thread operation interrupted", e) + Thread.currentThread().interrupt() + } + failure?.let { throw it } + return result + } + + fun serializeDrawing(): ByteArray? { + if (drawingEngine == 0L) return null + return runOnGlThreadSync(3000) { + if (drawingEngine != 0L) serializeDrawing(drawingEngine) else null + } + } + + fun deserializeDrawing(data: ByteArray): Boolean { + val success = runOnGlThreadSync { + if (drawingEngine != 0L) { deserializeDrawing(drawingEngine, data); true } else false + } ?: false + if (success) { + resetTransientInteractionState() + requestInkRender() + emitSelectionChange() + } + return success + } + + fun getBase64PngData(scale: Float): String? { + if (drawingEngine == 0L || viewWidth == 0 || viewHeight == 0) return null + return runOnGlThreadSync { exportToBase64(scale, Bitmap.CompressFormat.PNG, 100) } + } + + fun getBase64JpegData(scale: Float, compression: Float): String? { + if (drawingEngine == 0L || viewWidth == 0 || viewHeight == 0) return null + val quality = (compression * 100).toInt().coerceIn(0, 100) + return runOnGlThreadSync { exportToBase64(scale, Bitmap.CompressFormat.JPEG, quality) } + } + + private fun exportToBase64(scale: Float, format: Bitmap.CompressFormat, quality: Int): String? { + return try { + val actualScale = if (scale > 0f) scale else resources.displayMetrics.density + val width = (viewWidth * actualScale).toInt().coerceAtLeast(1) + val height = (viewHeight * actualScale).toInt().coerceAtLeast(1) + val finalBitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888) + if (drawingEngine != 0L) renderToPixelsScaled(drawingEngine, finalBitmap, actualScale) + + val stream = java.io.ByteArrayOutputStream() + finalBitmap.compress(format, quality, stream) + finalBitmap.recycle() + + val mimeType = if (format == Bitmap.CompressFormat.PNG) "png" else "jpeg" + "data:image/$mimeType;base64," + android.util.Base64.encodeToString(stream.toByteArray(), android.util.Base64.NO_WRAP) + } catch (e: Exception) { + android.util.Log.e("MobileInkCanvasView", "Export error", e) + null + } + } + + private fun sendEvent(eventName: String, params: com.facebook.react.bridge.WritableMap) { + val reactContext = context as ReactContext + reactContext + .getJSModule(RCTEventEmitter::class.java) + .receiveEvent(id, eventName, params) + } + + private fun sendDrawingBeginEvent(x: Float, y: Float) { + val payload = Arguments.createMap() + payload.putDouble("x", x.toDouble()) + payload.putDouble("y", y.toDouble()) + sendEvent("onDrawingBegin", payload) + } + + private fun emitSelectionChange() { + val payload = Arguments.createMap() + val bounds = getSelectionBounds() + val count = getSelectionCount() + payload.putInt("count", count) + if (count > 0 && bounds != null && bounds.size == 4) { + val boundsMap = Arguments.createMap() + boundsMap.putDouble("x", bounds[0].toDouble()) + boundsMap.putDouble("y", bounds[1].toDouble()) + boundsMap.putDouble("width", (bounds[2] - bounds[0]).toDouble()) + boundsMap.putDouble("height", (bounds[3] - bounds[1]).toDouble()) + payload.putMap("bounds", boundsMap) + } else { + payload.putNull("bounds") + } + sendEvent("onInkSelectionChange", payload) + } + + // Native method declarations + private external fun createDrawingEngine(width: Int, height: Int): Long + private external fun destroyDrawingEngine(engine: Long) + + // Touch handling with full stylus support + private external fun touchBegan(engine: Long, x: Float, y: Float, pressure: Float, azimuth: Float, altitude: Float, timestamp: Long, isStylusInput: Boolean) + private external fun touchMoved(engine: Long, x: Float, y: Float, pressure: Float, azimuth: Float, altitude: Float, timestamp: Long, isStylusInput: Boolean) + private external fun touchEnded(engine: Long, timestamp: Long) + private external fun updateHoldShapePreview(engine: Long, timestamp: Long): Boolean + + // Canvas operations + private external fun clearCanvas(engine: Long) + private external fun undoStroke(engine: Long) + private external fun redoStroke(engine: Long) + + // Tool settings + private external fun setStrokeColor(engine: Long, color: Int) + private external fun setStrokeWidth(engine: Long, width: Float) + private external fun setTool(engine: Long, toolType: String) + private external fun setToolWithParams(engine: Long, toolType: String, width: Float, color: Int, eraserMode: String) + private external fun setEraserCursor(engine: Long, x: Float, y: Float, radius: Float, visible: Boolean) + private external fun setBackgroundType(engine: Long, backgroundType: String) + private external fun setPdfBackgroundBitmap(engine: Long, bitmap: Bitmap?) + + // Selection operations + private external fun selectStrokeAt(engine: Long, x: Float, y: Float): Boolean + private external fun selectShapeStrokeAt(engine: Long, x: Float, y: Float): Boolean + private external fun clearSelection(engine: Long) + private external fun deleteSelection(engine: Long) + private external fun copySelection(engine: Long) + private external fun pasteSelection(engine: Long, offsetX: Float, offsetY: Float) + private external fun moveSelection(engine: Long, dx: Float, dy: Float) + private external fun finalizeMove(engine: Long) + private external fun beginSelectionTransform(engine: Long, handleIndex: Int) + private external fun updateSelectionTransform(engine: Long, x: Float, y: Float) + private external fun finalizeSelectionTransform(engine: Long) + private external fun cancelSelectionTransform(engine: Long) + private external fun getSelectionCount(engine: Long): Int + private external fun getSelectionBounds(engine: Long): FloatArray? + + // State queries + private external fun canUndo(engine: Long): Boolean + private external fun canRedo(engine: Long): Boolean + private external fun isEmpty(engine: Long): Boolean + + // Serialization + private external fun serializeDrawing(engine: Long): ByteArray? + private external fun deserializeDrawing(engine: Long, data: ByteArray) + + // Rendering + private external fun renderGaneshToCurrentSurface(engine: Long, width: Int, height: Int): Boolean + private external fun releaseGaneshContext(engine: Long) + private external fun renderToPixelsScaled(engine: Long, bitmap: Bitmap, scale: Float) + + companion object { + private var libraryLoaded = false + + @Synchronized + fun ensureLibraryLoaded(): Boolean { + if (!libraryLoaded) { + try { + System.loadLibrary("mobileink") + libraryLoaded = true + } catch (e: UnsatisfiedLinkError) { + android.util.Log.e("MobileInkCanvasView", "Failed to load mobileink library: ${e.message}") + return false + } + } + return libraryLoaded + } + } +} diff --git a/android/src/main/java/com/mathnotes/mobileink/MobileInkCanvasViewManager.kt b/android/src/main/java/com/mathnotes/mobileink/MobileInkCanvasViewManager.kt new file mode 100644 index 0000000..3a0e77e --- /dev/null +++ b/android/src/main/java/com/mathnotes/mobileink/MobileInkCanvasViewManager.kt @@ -0,0 +1,174 @@ +package com.mathnotes.mobileink + +import android.graphics.Color +import com.facebook.react.bridge.ReactApplicationContext +import com.facebook.react.bridge.ReadableArray +import com.facebook.react.uimanager.SimpleViewManager +import com.facebook.react.uimanager.ThemedReactContext +import com.facebook.react.uimanager.annotations.ReactProp + +/** + * MobileInkCanvasViewManager - manages the MobileInkCanvasView which handles drawing. + * Background patterns are rendered in the C++ Skia layer, not as a separate Android View. + * This ensures eraser works correctly and view registration is proper. + */ +class MobileInkCanvasViewManager(private val reactContext: ReactApplicationContext) : + SimpleViewManager() { + + override fun getName() = "MobileInkCanvasView" + + override fun createViewInstance(reactContext: ThemedReactContext): MobileInkCanvasView { + return MobileInkCanvasView(reactContext) + } + + override fun getExportedCustomBubblingEventTypeConstants(): Map { + return mapOf( + "onDrawingChange" to mapOf( + "phasedRegistrationNames" to mapOf("bubbled" to "onDrawingChange") + ), + "onDrawingBegin" to mapOf( + "phasedRegistrationNames" to mapOf("bubbled" to "onDrawingBegin") + ), + "onInkSelectionChange" to mapOf( + "phasedRegistrationNames" to mapOf("bubbled" to "onInkSelectionChange") + ) + ) + } + + @ReactProp(name = "backgroundColor", customType = "Color") + fun setDrawingBackgroundColor(view: MobileInkCanvasView, color: Int) { + view.setDrawingBackgroundColor(color) + } + + @ReactProp(name = "backgroundType") + fun setBackgroundType(view: MobileInkCanvasView, type: String?) { + view.setBackgroundType(type ?: "plain") + } + + @ReactProp(name = "pdfBackgroundUri") + fun setPdfBackgroundUri(view: MobileInkCanvasView, uri: String?) { + view.setPdfBackgroundUri(uri) + } + + @ReactProp(name = "drawingPolicy") + fun setDrawingPolicy(view: MobileInkCanvasView, policy: String?) { + view.drawingPolicy = policy ?: "default" + } + + @ReactProp(name = "renderSuspended") + fun setRenderSuspended(view: MobileInkCanvasView, suspended: Boolean) { + view.renderSuspended = suspended + } + + @ReactProp(name = "renderBackend") + fun setRenderBackend(view: MobileInkCanvasView, backend: String?) { + view.renderBackend = backend ?: "ganesh" + } + + private fun parseToolColor(colorHex: String): Int { + val trimmed = colorHex.trim() + val hex = trimmed.removePrefix("#") + if (hex.length == 8 && hex.all { it in '0'..'9' || it in 'a'..'f' || it in 'A'..'F' }) { + val value = hex.toLong(16) + val red = ((value shr 24) and 0xFF).toInt() + val green = ((value shr 16) and 0xFF).toInt() + val blue = ((value shr 8) and 0xFF).toInt() + val alpha = (value and 0xFF).toInt() + return Color.argb(alpha, red, green, blue) + } + + return Color.parseColor(trimmed) + } + + override fun receiveCommand( + root: MobileInkCanvasView, + commandId: String, + args: ReadableArray? + ) { + when (commandId) { + "clear" -> root.clear() + "undo" -> root.undo() + "redo" -> root.redo() + "setTool" -> { + if (args != null && args.size() >= 3) { + val toolType = args.getString(0) ?: "pen" + val width = args.getDouble(1).toFloat() + val colorHex = args.getString(2) ?: "#000000" + val color = parseToolColor(colorHex) + root.setTool(toolType, width, color) + } + } + "setToolWithParams" -> { + if (args != null && args.size() >= 3) { + val toolType = args.getString(0) ?: "pen" + val width = args.getDouble(1).toFloat() + val colorHex = args.getString(2) ?: "#000000" + val color = parseToolColor(colorHex) + val eraserMode = if (args.size() > 3) args.getString(3) else null + root.setToolWithParams(toolType, width, color, eraserMode) + } + } + "selectAt" -> { + if (args != null && args.size() >= 2) { + val x = args.getDouble(0).toFloat() + val y = args.getDouble(1).toFloat() + root.selectAt(x, y) + } + } + "clearSelection" -> root.clearSelection() + "deleteSelection" -> root.deleteSelection() + "copySelection" -> root.copySelection() + // Aliases used by JS side + "performCopy" -> root.copySelection() + "performPaste" -> root.pasteSelection(50f, 50f) + "performDelete" -> root.deleteSelection() + "pasteSelection" -> { + if (args != null && args.size() >= 2) { + val offsetX = args.getDouble(0).toFloat() + val offsetY = args.getDouble(1).toFloat() + root.pasteSelection(offsetX, offsetY) + } + } + "moveSelection" -> { + if (args != null && args.size() >= 2) { + val dx = args.getDouble(0).toFloat() + val dy = args.getDouble(1).toFloat() + root.moveSelection(dx, dy) + } + } + "finalizeMove" -> root.finalizeMoveSelection() + "deserializeDrawing" -> { + // Deserialize from JSON string (format: {"pages":{"0":""}}) + // Match iOS behavior: clear canvas when data is empty/missing + if (args != null && args.size() >= 1) { + val jsonString = args.getString(0) + if (jsonString != null) { + try { + // Parse JSON to extract base64 from pages.0 + val json = org.json.JSONObject(jsonString) + val pages = json.optJSONObject("pages") + val base64 = pages?.optString("0") + if (!base64.isNullOrEmpty()) { + val data = android.util.Base64.decode(base64, android.util.Base64.DEFAULT) + root.deserializeDrawing(data) + } else { + // Match iOS: clear canvas when no data for page 0 + root.clearCanvasAsyncForLoad() + } + } catch (e: Exception) { + // Clear on parse error to prevent stale data (match iOS behavior) + root.clearCanvasAsyncForLoad() + android.util.Log.e("MobileInkCanvasViewManager", "Failed to deserialize drawing", e) + } + } else { + // Clear when jsonString is null + root.clearCanvasAsyncForLoad() + } + } else { + // Clear when no args provided + root.clearCanvasAsyncForLoad() + } + } + } + } +} diff --git a/android/src/main/java/com/mathnotes/mobileink/MobileInkModule.kt b/android/src/main/java/com/mathnotes/mobileink/MobileInkModule.kt new file mode 100644 index 0000000..f6ec879 --- /dev/null +++ b/android/src/main/java/com/mathnotes/mobileink/MobileInkModule.kt @@ -0,0 +1,656 @@ +package com.mathnotes.mobileink + +import android.net.Uri +import android.graphics.Bitmap +import android.util.Base64 +import com.facebook.react.bridge.Promise +import com.facebook.react.bridge.ReactApplicationContext +import com.facebook.react.bridge.ReactContextBaseJavaModule +import com.facebook.react.bridge.ReactMethod +import com.facebook.react.bridge.ReadableArray +import com.facebook.react.bridge.WritableArray +import com.facebook.react.bridge.WritableMap +import com.facebook.react.bridge.Arguments +import com.facebook.react.module.annotations.ReactModule +import org.json.JSONArray +import org.json.JSONObject +import java.io.File +import java.lang.ref.WeakReference +import java.util.concurrent.Executors + +/** + * MobileInkModule provides callback-based APIs for the drawing canvas. + * Used for operations that need to return data (like serialization, state queries). + */ +@ReactModule(name = MobileInkModule.NAME) +class MobileInkModule(private val reactContext: ReactApplicationContext) : + ReactContextBaseJavaModule(reactContext) { + + companion object { + const val NAME = "MobileInkModule" + + // Thread pool for background processing + private val executor = Executors.newSingleThreadExecutor() + + // Static registry of MobileInkCanvasView instances by their React tag + // Uses WeakReference to avoid memory leaks when views are unmounted + private val viewRegistry = mutableMapOf>() + + /** + * Register a view when it's created (called from MobileInkCanvasView) + */ + fun registerView(tag: Int, view: MobileInkCanvasView) { + viewRegistry[tag] = WeakReference(view) + } + + /** + * Unregister a view when it's destroyed (called from MobileInkCanvasView) + */ + fun unregisterView(tag: Int) { + viewRegistry.remove(tag) + } + + /** + * Get a view by its React tag + */ + fun getView(tag: Int): MobileInkCanvasView? { + return viewRegistry[tag]?.get() + } + + // Native method for batch export - static because it creates its own temp engine + @JvmStatic + private external fun nativeBatchExportPages( + pagesDataArray: Array, + backgroundTypes: Array, + pdfBackgrounds: Array, + pageIndices: IntArray, + width: Int, + height: Int, + scale: Float + ): Array + + @JvmStatic + private external fun nativeComposeContinuousWindow( + pageDataArray: Array, + pageHeight: Float + ): ByteArray? + + @JvmStatic + private external fun nativeDecomposeContinuousWindow( + windowData: ByteArray?, + pageCount: Int, + pageHeight: Float + ): Array + } + + override fun getName() = NAME + + private fun findDrawingView(viewTag: Int): MobileInkCanvasView? { + return getView(viewTag) + } + + private fun fileFromBridgePath(filePath: String): File { + if (filePath.startsWith("file://")) { + val uriPath = Uri.parse(filePath).path + if (!uriPath.isNullOrEmpty()) { + return File(uriPath) + } + return File(filePath.removePrefix("file://")) + } + return File(filePath) + } + + @ReactMethod + fun getBase64Data(viewTag: Int, promise: Promise) { + reactContext.runOnUiQueueThread { + try { + val view = findDrawingView(viewTag) + if (view != null) { + val data = view.serializeDrawing() + if (data != null && data.isNotEmpty()) { + val base64 = Base64.encodeToString(data, Base64.NO_WRAP) + // Wrap in JSON format like iOS: {"pages":{"0":""}} + val json = """{"pages":{"0":"$base64"}}""" + promise.resolve(json) + } else { + // Empty drawing - return empty JSON like iOS + promise.resolve("""{"pages":{}}""") + } + } else { + // View not found - likely unmounted during auto-save, return null gracefully + android.util.Log.w("MobileInkModule", "getBase64Data: view not found for tag $viewTag") + promise.resolve(null) + } + } catch (e: Exception) { + // Don't reject on exceptions either - gracefully return null + android.util.Log.e("MobileInkModule", "getBase64Data: exception", e) + promise.resolve(null) + } + } + } + + @ReactMethod + fun canUndo(viewTag: Int, promise: Promise) { + reactContext.runOnUiQueueThread { + try { + val view = findDrawingView(viewTag) + promise.resolve(view?.canUndo() ?: false) + } catch (e: Exception) { + promise.reject("ERROR", e.message) + } + } + } + + @ReactMethod + fun canRedo(viewTag: Int, promise: Promise) { + reactContext.runOnUiQueueThread { + try { + val view = findDrawingView(viewTag) + promise.resolve(view?.canRedo() ?: false) + } catch (e: Exception) { + promise.reject("ERROR", e.message) + } + } + } + + @ReactMethod + fun isEmpty(viewTag: Int, promise: Promise) { + reactContext.runOnUiQueueThread { + try { + val view = findDrawingView(viewTag) + promise.resolve(view?.isEmpty() ?: true) + } catch (e: Exception) { + promise.reject("ERROR", e.message) + } + } + } + + @ReactMethod + fun getSelectionCount(viewTag: Int, promise: Promise) { + reactContext.runOnUiQueueThread { + try { + val view = findDrawingView(viewTag) + promise.resolve(view?.getSelectionCount() ?: 0) + } catch (e: Exception) { + promise.reject("ERROR", e.message) + } + } + } + + @ReactMethod + fun getSelectionBounds(viewTag: Int, promise: Promise) { + reactContext.runOnUiQueueThread { + try { + val view = findDrawingView(viewTag) + val bounds = view?.getSelectionBounds() + if (bounds != null && bounds.size == 4) { + val result = Arguments.createMap() + result.putDouble("minX", bounds[0].toDouble()) + result.putDouble("minY", bounds[1].toDouble()) + result.putDouble("maxX", bounds[2].toDouble()) + result.putDouble("maxY", bounds[3].toDouble()) + promise.resolve(result) + } else { + promise.resolve(null) + } + } catch (e: Exception) { + promise.reject("ERROR", e.message) + } + } + } + + @ReactMethod + fun getBase64PngData(viewTag: Int, scale: Double, promise: Promise) { + reactContext.runOnUiQueueThread { + try { + val view = findDrawingView(viewTag) + if (view != null) { + val data = view.getBase64PngData(scale.toFloat()) + if (data != null) { + promise.resolve(data) + } else { + promise.resolve("") + } + } else { + android.util.Log.w("MobileInkModule", "getBase64PngData: view not found for tag $viewTag") + promise.resolve("") + } + } catch (e: Exception) { + android.util.Log.e("MobileInkModule", "getBase64PngData: exception", e) + promise.resolve("") + } + } + } + + @ReactMethod + fun getBase64JpegData(viewTag: Int, scale: Double, compression: Double, promise: Promise) { + reactContext.runOnUiQueueThread { + try { + val view = findDrawingView(viewTag) + if (view != null) { + val data = view.getBase64JpegData(scale.toFloat(), compression.toFloat()) + if (data != null) { + promise.resolve(data) + } else { + promise.resolve("") + } + } else { + android.util.Log.w("MobileInkModule", "getBase64JpegData: view not found for tag $viewTag") + promise.resolve("") + } + } catch (e: Exception) { + android.util.Log.e("MobileInkModule", "getBase64JpegData: exception", e) + promise.resolve("") + } + } + } + + /** + * Load base64 drawing data into the canvas. + * Returns a Promise that resolves to true when loading is complete. + * This ensures the caller can wait for the load to finish before proceeding. + */ + @ReactMethod + fun loadBase64Data(viewTag: Int, jsonString: String, promise: Promise) { + reactContext.runOnUiQueueThread { + try { + val view = findDrawingView(viewTag) + if (view == null) { + android.util.Log.w("MobileInkModule", "loadBase64Data: view not found for tag $viewTag") + promise.resolve(false) + return@runOnUiQueueThread + } + + // Parse JSON format: {"pages":{"0":""}} + val json = JSONObject(jsonString) + val pages = json.optJSONObject("pages") + if (pages == null) { + promise.resolve(view.clearCanvasForLoad()) + return@runOnUiQueueThread + } + + // Get page 0 data (current page) + val base64 = pages.optString("0", "") + if (base64.isEmpty()) { + promise.resolve(view.clearCanvasForLoad()) + return@runOnUiQueueThread + } + + // Decode and load + val data = Base64.decode(base64, Base64.DEFAULT) + + // deserializeDrawing is now synchronous (waits for GL thread) + val success = view.deserializeDrawing(data) + promise.resolve(success) + } catch (e: Exception) { + android.util.Log.e("MobileInkModule", "loadBase64Data: exception", e) + promise.resolve(false) + } + } + } + + /** + * Native-side persistence: serialize the engine's current state to base64 JSON + * and write directly to disk. The body never crosses the JS<->native bridge -- + * a major drawing-time win on heavy notebooks where the body is multi-MB. + * + * Atomicity: write goes to {path}.tmp first, then rename to {path}. POSIX + * rename is atomic on the same filesystem so a partial write under memory + * pressure can never poison the canonical body file. + */ + @ReactMethod + fun persistEngineToFile(viewTag: Int, filePath: String, promise: Promise) { + reactContext.runOnUiQueueThread { + val view = findDrawingView(viewTag) + if (view == null) { + android.util.Log.w("MobileInkModule", "persistEngineToFile: view not found for tag $viewTag") + promise.reject("VIEW_NOT_FOUND", "MobileInkCanvasView not found") + return@runOnUiQueueThread + } + + // Serialize on the UI queue (deserialize/serialize already coordinate + // with the GL thread inside the view), then move the file write to a + // background executor so it doesn't block the UI thread. + val data: ByteArray? = try { + view.serializeDrawing() + } catch (e: Exception) { + android.util.Log.e("MobileInkModule", "persistEngineToFile: serialize exception", e) + promise.reject("GET_DATA_ERROR", e.message ?: "serialize failed", e) + return@runOnUiQueueThread + } + + executor.execute { + try { + val body = if (data != null && data.isNotEmpty()) { + val base64 = Base64.encodeToString(data, Base64.NO_WRAP) + """{"pages":{"0":"$base64"}}""" + } else { + """{"pages":{}}""" + } + + val target = fileFromBridgePath(filePath) + target.parentFile?.let { parent -> + if (!parent.exists()) parent.mkdirs() + } + val tmp = File("${target.absolutePath}.tmp") + // Best-effort: drop a leftover .tmp from a previous failed write. + if (tmp.exists()) tmp.delete() + tmp.writeText(body) + val renamed = tmp.renameTo(target) + if (!renamed) { + // Rename failed -- fall back to a regular write into target, + // then drop the .tmp. Loses atomicity but at least the body + // is durable on disk. + target.writeText(body) + if (tmp.exists()) tmp.delete() + } + promise.resolve(true) + } catch (e: Exception) { + android.util.Log.e("MobileInkModule", "persistEngineToFile: write exception", e) + promise.reject("WRITE_FAILED", e.message ?: "write failed", e) + } + } + } + } + + /** + * Inverse of persistEngineToFile: read the body file directly and feed it + * into the engine without the bytes ever crossing the bridge as a JS string. + * + * Resolves true on successful load, false if the file is missing/empty. + * Rejects only on actual deserialization errors. + */ + @ReactMethod + fun loadEngineFromFile(viewTag: Int, filePath: String, promise: Promise) { + executor.execute { + val target = fileFromBridgePath(filePath) + if (!target.exists() || target.length() == 0L) { + promise.resolve(false) + return@execute + } + + val body: String = try { + target.readText() + } catch (e: Exception) { + android.util.Log.e("MobileInkModule", "loadEngineFromFile: read exception", e) + promise.resolve(false) + return@execute + } + + reactContext.runOnUiQueueThread { + try { + val view = findDrawingView(viewTag) + if (view == null) { + promise.reject("VIEW_NOT_FOUND", "MobileInkCanvasView not found") + return@runOnUiQueueThread + } + + val json = JSONObject(body) + val pages = json.optJSONObject("pages") + if (pages == null) { + promise.resolve(view.clearCanvasForLoad()) + return@runOnUiQueueThread + } + + val base64 = pages.optString("0", "") + if (base64.isEmpty()) { + promise.resolve(view.clearCanvasForLoad()) + return@runOnUiQueueThread + } + + val data = Base64.decode(base64, Base64.DEFAULT) + val success = view.deserializeDrawing(data) + promise.resolve(success) + } catch (e: Exception) { + android.util.Log.e("MobileInkModule", "loadEngineFromFile: deserialize exception", e) + promise.reject("LOAD_DATA_ERROR", e.message ?: "deserialize failed", e) + } + } + } + } + + /** + * Batch export multiple pages to PNG images. + * This is much faster than exporting pages one by one because it: + * 1. Creates a single Skia engine and surface (reused for all pages) + * 2. Doesn't switch visible pages (no UI updates) + * 3. Processes all pages in a single native call + * + * @param pagesDataArray Array of JSON strings with format {"pages":{"0":""}} + * @param backgroundTypes Array of background type strings ("plain", "lined", "grid", "pdf") + * @param width Canvas width in pixels + * @param height Canvas height in pixels + * @param scale Export scale factor (e.g., 2.0 for retina) + * @param pdfBackgroundUri Optional PDF file URI for PDF backgrounds. + * @param pageIndices Original notebook page indices for page-aware background export. + * @param promise Resolves to array of base64 PNG data URIs + */ + @ReactMethod + fun batchExportPages( + pagesDataArray: ReadableArray, + backgroundTypes: ReadableArray, + width: Int, + height: Int, + scale: Double, + pdfBackgroundUri: String, + pageIndices: ReadableArray, + promise: Promise + ) { + // Ensure native library is loaded + if (!MobileInkCanvasView.ensureLibraryLoaded()) { + promise.reject("LIBRARY_NOT_LOADED", "Failed to load native drawing library") + return + } + + // Process on background thread to avoid blocking UI + executor.execute { + try { + val numPages = pagesDataArray.size() + if (numPages == 0) { + promise.resolve(Arguments.createArray()) + return@execute + } + + // Decode page payloads before JNI so valid JSON formatting differences + // do not leak into the C++ boundary. + val pagesData = Array(numPages) { i -> + decodePagePayload(pagesDataArray.getString(i) ?: "") + } + val bgTypes = Array(numPages) { i -> + if (i < backgroundTypes.size()) backgroundTypes.getString(i) ?: "plain" else "plain" + } + val resolvedPageIndices = IntArray(numPages) { i -> + if (i < pageIndices.size()) pageIndices.getInt(i) else i + } + val pdfBackgrounds = Array(numPages) { i -> + if (bgTypes[i] != "pdf" || pdfBackgroundUri.isEmpty()) { + null + } else { + val pageAwareUri = pdfUriForPage(pdfBackgroundUri, resolvedPageIndices[i], numPages) + PdfLoader.loadAndRenderPdf(reactApplicationContext, pageAwareUri, width, height) + } + } + + // Call native batch export + val results = nativeBatchExportPages( + pagesData, + bgTypes, + pdfBackgrounds, + resolvedPageIndices, + width, + height, + scale.toFloat() + ) + + pdfBackgrounds.forEach { it?.recycle() } + + // Convert to WritableArray + val resultArray = Arguments.createArray() + for (result in results) { + resultArray.pushString(result ?: "") + } + + promise.resolve(resultArray) + } catch (e: Exception) { + android.util.Log.e("MobileInkModule", "batchExportPages: exception", e) + promise.reject("BATCH_EXPORT_ERROR", e.message, e) + } + } + } + + /** + * Native body-file read + parse. Mirrors the iOS helper so Android callers + * can skip a multi-MB JS string read/JSON.parse when opening large notebooks. + */ + @ReactMethod + fun readBodyFileParsed(bodyPath: String, promise: Promise) { + executor.execute { + val target = fileFromBridgePath(bodyPath) + if (!target.exists() || target.length() == 0L) { + promise.resolve(null) + return@execute + } + + try { + val parsed = JSONObject(target.readText()) + promise.resolve(jsonObjectToWritableMap(parsed)) + } catch (e: Exception) { + android.util.Log.e("MobileInkModule", "readBodyFileParsed: exception", e) + promise.reject("READ_PARSE_FAILED", e.message ?: "read/parse failed", e) + } + } + } + + @ReactMethod + fun composeContinuousWindow(pagePayloads: ReadableArray, pageHeight: Double, promise: Promise) { + if (!MobileInkCanvasView.ensureLibraryLoaded()) { + promise.reject("LIBRARY_NOT_LOADED", "Failed to load native drawing library") + return + } + + executor.execute { + try { + val pageData = Array(pagePayloads.size()) { i -> + decodePagePayload(pagePayloads.getString(i) ?: "") + } + val combinedData = nativeComposeContinuousWindow(pageData, pageHeight.toFloat()) + promise.resolve(makePagePayload(combinedData)) + } catch (e: Exception) { + android.util.Log.e("MobileInkModule", "composeContinuousWindow: exception", e) + promise.reject("COMPOSE_FAILED", e.message ?: "compose failed", e) + } + } + } + + @ReactMethod + fun decomposeContinuousWindow( + windowPayload: String, + pageCount: Int, + pageHeight: Double, + promise: Promise + ) { + if (!MobileInkCanvasView.ensureLibraryLoaded()) { + promise.reject("LIBRARY_NOT_LOADED", "Failed to load native drawing library") + return + } + + executor.execute { + try { + val windowData = decodePagePayload(windowPayload) + val pageData = nativeDecomposeContinuousWindow(windowData, pageCount, pageHeight.toFloat()) + val result = Arguments.createArray() + for (i in 0 until pageCount) { + result.pushString(makePagePayload(pageData.getOrNull(i))) + } + promise.resolve(result) + } catch (e: Exception) { + android.util.Log.e("MobileInkModule", "decomposeContinuousWindow: exception", e) + promise.reject("DECOMPOSE_FAILED", e.message ?: "decompose failed", e) + } + } + } + + private fun pdfUriForPage(pdfBackgroundUri: String, pageIndex: Int, exportedPageCount: Int): String { + if (exportedPageCount == 1 && pdfBackgroundUri.contains("#page=")) { + return pdfBackgroundUri + } + val cleanUri = pdfBackgroundUri.substringBefore("#") + return "$cleanUri#page=${pageIndex + 1}" + } + + private fun makePagePayload(data: ByteArray?): String { + if (data == null || data.isEmpty()) { + return """{"pages":{}}""" + } + val base64 = Base64.encodeToString(data, Base64.NO_WRAP) + return """{"pages":{"0":"$base64"}}""" + } + + private fun decodePagePayload(pageJson: String): ByteArray? { + return try { + val json = JSONObject(pageJson) + val pages = json.optJSONObject("pages") ?: return null + val base64 = pages.optString("0", "") + if (base64.isEmpty()) { + null + } else { + Base64.decode(base64, Base64.DEFAULT).takeIf { it.isNotEmpty() } + } + } catch (e: Exception) { + android.util.Log.w("MobileInkModule", "decodePagePayload: treating invalid page payload as blank") + null + } + } + + private fun jsonObjectToWritableMap(json: JSONObject): WritableMap { + val map = Arguments.createMap() + val keys = json.keys() + while (keys.hasNext()) { + val key = keys.next() + putJsonValue(map, key, json.opt(key)) + } + return map + } + + private fun jsonArrayToWritableArray(json: JSONArray): WritableArray { + val array = Arguments.createArray() + for (i in 0 until json.length()) { + when (val value = json.opt(i)) { + null, JSONObject.NULL -> array.pushNull() + is JSONObject -> array.pushMap(jsonObjectToWritableMap(value)) + is JSONArray -> array.pushArray(jsonArrayToWritableArray(value)) + is Boolean -> array.pushBoolean(value) + is Int -> array.pushInt(value) + is Long -> { + if (value >= Int.MIN_VALUE && value <= Int.MAX_VALUE) { + array.pushInt(value.toInt()) + } else { + array.pushDouble(value.toDouble()) + } + } + is Number -> array.pushDouble(value.toDouble()) + else -> array.pushString(value.toString()) + } + } + return array + } + + private fun putJsonValue(map: WritableMap, key: String, value: Any?) { + when (value) { + null, JSONObject.NULL -> map.putNull(key) + is JSONObject -> map.putMap(key, jsonObjectToWritableMap(value)) + is JSONArray -> map.putArray(key, jsonArrayToWritableArray(value)) + is Boolean -> map.putBoolean(key, value) + is Int -> map.putInt(key, value) + is Long -> { + if (value >= Int.MIN_VALUE && value <= Int.MAX_VALUE) { + map.putInt(key, value.toInt()) + } else { + map.putDouble(key, value.toDouble()) + } + } + is Number -> map.putDouble(key, value.toDouble()) + else -> map.putString(key, value.toString()) + } + } +} diff --git a/android/src/main/java/com/mathnotes/mobileink/MobileInkPackage.kt b/android/src/main/java/com/mathnotes/mobileink/MobileInkPackage.kt new file mode 100644 index 0000000..c5aae1e --- /dev/null +++ b/android/src/main/java/com/mathnotes/mobileink/MobileInkPackage.kt @@ -0,0 +1,51 @@ +package com.mathnotes.mobileink + +import com.facebook.react.TurboReactPackage +import com.facebook.react.bridge.NativeModule +import com.facebook.react.bridge.ReactApplicationContext +import com.facebook.react.module.annotations.ReactModuleList +import com.facebook.react.module.model.ReactModuleInfo +import com.facebook.react.module.model.ReactModuleInfoProvider +import com.facebook.react.uimanager.ViewManager + +/** + * TurboReactPackage for the native drawing module. + * + * Using TurboReactPackage instead of ReactPackage is critical for React Native 0.81+ + * with new architecture enabled. It provides module metadata via getReactModuleInfoProvider() + * which allows RN to register modules without loading native libraries during startup scan. + */ +@ReactModuleList(nativeModules = [MobileInkModule::class]) +class MobileInkPackage : TurboReactPackage() { + + override fun createNativeModules(reactContext: ReactApplicationContext): List { + return listOf(MobileInkModule(reactContext)) + } + + override fun getModule(name: String, reactContext: ReactApplicationContext): NativeModule? { + return when (name) { + MobileInkModule.NAME -> MobileInkModule(reactContext) + else -> null + } + } + + override fun getReactModuleInfoProvider(): ReactModuleInfoProvider { + return ReactModuleInfoProvider { + mapOf( + MobileInkModule.NAME to ReactModuleInfo( + MobileInkModule.NAME, + MobileInkModule::class.java.name, + false, // canOverrideExistingModule + false, // needsEagerInit + false, // hasConstants - we have no getConstants() + false, // isCxxModule - this is Kotlin, not C++ + false // isTurboModule - false to use legacy NativeModules bridge + ) + ) + } + } + + override fun createViewManagers(reactContext: ReactApplicationContext): List> { + return listOf(MobileInkCanvasViewManager(reactContext)) + } +} diff --git a/android/src/main/java/com/mathnotes/mobileink/PdfLoader.kt b/android/src/main/java/com/mathnotes/mobileink/PdfLoader.kt new file mode 100644 index 0000000..09e943b --- /dev/null +++ b/android/src/main/java/com/mathnotes/mobileink/PdfLoader.kt @@ -0,0 +1,204 @@ +package com.mathnotes.mobileink + +import android.content.Context +import android.graphics.Bitmap +import android.graphics.Matrix +import android.graphics.pdf.PdfRenderer +import android.net.Uri +import android.os.ParcelFileDescriptor +import android.util.Base64 +import java.io.File +import java.net.HttpURLConnection +import java.net.URL + +/** + * Utility class for loading PDF files from various sources and rendering them to bitmaps. + * Supports: content:// (document picker), file://, https://, http://, + * absolute paths, and data:application/pdf;base64 URIs. + */ +object PdfLoader { + + /** + * Load a PDF and render it to a bitmap. + * @param context Android context for content resolver access + * @param uri PDF URI (supports multiple formats) + * @param viewWidth Target width for rendering (uses PDF native width if 0) + * @param viewHeight Target height for rendering (uses PDF native height if 0) + * @return Rendered bitmap or null on failure + */ + fun loadAndRenderPdf(context: Context, uri: String, viewWidth: Int, viewHeight: Int): Bitmap? { + // Parse page number from URI fragment (#page=N, 1-indexed) + var pageNumber = 0 + var cleanUri = uri + + val hashIndex = uri.indexOf('#') + if (hashIndex != -1) { + val fragment = uri.substring(hashIndex + 1) + if (fragment.startsWith("page=")) { + pageNumber = (fragment.substringAfter("page=").toIntOrNull() ?: 1) - 1 + } + cleanUri = uri.substring(0, hashIndex) + } + + return try { + when { + // Handle base64 data URI + cleanUri.startsWith("data:application/pdf;base64,") -> { + loadPdfFromBase64(context, cleanUri, pageNumber, viewWidth, viewHeight) + } + // Handle content:// URI (from document picker - Google Drive, Downloads, etc.) + cleanUri.startsWith("content://") -> { + loadPdfFromContentUri(context, cleanUri, pageNumber, viewWidth, viewHeight) + } + // Handle https:// and http:// URLs + cleanUri.startsWith("https://") || cleanUri.startsWith("http://") -> { + loadPdfFromUrl(context, cleanUri, pageNumber, viewWidth, viewHeight) + } + // Handle file:// URI + cleanUri.startsWith("file://") -> { + loadPdfFromFile(File(cleanUri.removePrefix("file://")), pageNumber, viewWidth, viewHeight) + } + // Handle absolute path + cleanUri.startsWith("/") -> { + loadPdfFromFile(File(cleanUri), pageNumber, viewWidth, viewHeight) + } + else -> { + android.util.Log.e("PdfLoader", "Unsupported PDF URI format") + null + } + } + } catch (e: Exception) { + android.util.Log.e("PdfLoader", "Error loading PDF", e) + null + } + } + + /** + * Load PDF from a base64 data URI by writing to temp file. + */ + private fun loadPdfFromBase64(context: Context, dataUri: String, pageNumber: Int, viewWidth: Int, viewHeight: Int): Bitmap? { + val base64Data = dataUri.removePrefix("data:application/pdf;base64,") + val pdfBytes = Base64.decode(base64Data, Base64.DEFAULT) + + // PdfRenderer requires a file, so write to temp file + val tempFile = File.createTempFile("pdf_bg_", ".pdf", context.cacheDir) + try { + tempFile.writeBytes(pdfBytes) + return loadPdfFromFile(tempFile, pageNumber, viewWidth, viewHeight) + } finally { + tempFile.delete() + } + } + + /** + * Load PDF from a file and render the specified page to a bitmap. + */ + private fun loadPdfFromFile(file: File, pageNumber: Int, viewWidth: Int, viewHeight: Int): Bitmap? { + if (!file.exists()) { + android.util.Log.e("PdfLoader", "PDF file not found") + return null + } + + val pfd = ParcelFileDescriptor.open(file, ParcelFileDescriptor.MODE_READ_ONLY) + val renderer = PdfRenderer(pfd) + + try { + // Clamp page number to valid range + val validPageNumber = pageNumber.coerceIn(0, renderer.pageCount - 1) + val page = renderer.openPage(validPageNumber) + + try { + return renderPdfPage(page, viewWidth, viewHeight) + } finally { + page.close() + } + } finally { + renderer.close() + pfd.close() + } + } + + /** + * Load PDF from a content:// URI (from Android document picker). + */ + private fun loadPdfFromContentUri(context: Context, uriString: String, pageNumber: Int, viewWidth: Int, viewHeight: Int): Bitmap? { + val uri = Uri.parse(uriString) + + val pfd = context.contentResolver.openFileDescriptor(uri, "r") + ?: run { + android.util.Log.e("PdfLoader", "Failed to open content URI") + return null + } + + val renderer = PdfRenderer(pfd) + + try { + val validPageNumber = pageNumber.coerceIn(0, renderer.pageCount - 1) + val page = renderer.openPage(validPageNumber) + + try { + return renderPdfPage(page, viewWidth, viewHeight) + } finally { + page.close() + } + } finally { + renderer.close() + pfd.close() + } + } + + /** + * Load PDF from an https:// or http:// URL by downloading to temp file. + */ + private fun loadPdfFromUrl(context: Context, urlString: String, pageNumber: Int, viewWidth: Int, viewHeight: Int): Bitmap? { + val tempFile = File.createTempFile("pdf_download_", ".pdf", context.cacheDir) + try { + val connection = URL(urlString).openConnection() as HttpURLConnection + connection.connectTimeout = 30000 + connection.readTimeout = 30000 + connection.instanceFollowRedirects = true + + if (connection.responseCode != HttpURLConnection.HTTP_OK) { + android.util.Log.e("PdfLoader", + "Failed to download PDF: HTTP ${connection.responseCode}") + return null + } + + connection.inputStream.use { input -> + tempFile.outputStream().use { output -> + input.copyTo(output) + } + } + + return loadPdfFromFile(tempFile, pageNumber, viewWidth, viewHeight) + } catch (e: Exception) { + android.util.Log.e("PdfLoader", "Error downloading PDF", e) + return null + } finally { + tempFile.delete() + } + } + + private fun renderPdfPage(page: PdfRenderer.Page, viewWidth: Int, viewHeight: Int): Bitmap { + val targetWidth = if (viewWidth > 0) viewWidth else page.width + val targetHeight = if (viewHeight > 0) viewHeight else page.height + val bitmap = Bitmap.createBitmap(targetWidth, targetHeight, Bitmap.Config.ARGB_8888) + bitmap.eraseColor(android.graphics.Color.WHITE) + + val fitScale = minOf( + targetWidth.toFloat() / page.width.toFloat(), + targetHeight.toFloat() / page.height.toFloat() + ) + val scaledWidth = page.width * fitScale + val scaledHeight = page.height * fitScale + val offsetX = (targetWidth - scaledWidth) / 2f + val offsetY = (targetHeight - scaledHeight) / 2f + val matrix = Matrix().apply { + postScale(fitScale, fitScale) + postTranslate(offsetX, offsetY) + } + + page.render(bitmap, null, matrix, PdfRenderer.Page.RENDER_MODE_FOR_DISPLAY) + return bitmap + } +} diff --git a/docs/api.md b/docs/api.md index 3fe56a5..618da2e 100644 --- a/docs/api.md +++ b/docs/api.md @@ -32,7 +32,7 @@ High-level continuous notebook component. - `onPagesChange(pages)`: called when page growth or trimming changes the page array. - `onSelectionChange(pageId, count, bounds)`: native selection event by page. - `onMotionStateChange(isMoving)`: viewport gesture/momentum state. -- `onPencilDoubleTap(event)`: Apple Pencil double-tap callback. +- `onPencilDoubleTap(event)`: Apple Pencil double-tap callback on iOS. ### Ref diff --git a/docs/architecture.md b/docs/architecture.md index 2a25009..f3b36f7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -24,12 +24,14 @@ flowchart TD - `ContinuousEnginePool` keeps a fixed number of native canvases mounted and reassigns them to pages. - `InfiniteInkCanvas` composes the viewport and pool into a generic continuous notebook shell. -### iOS native bridge +### Native bridges -- `MobileInkCanvasView` is the Metal-backed native drawing surface. -- `MobileInkCanvasViewManager` exposes React Native props and commands. -- `MobileInkBridge` exposes batch export, notebook parsing, and persistence helpers that are not tied to a single view. -- `MobileInkBackgroundView` renders generic page backgrounds. +- iOS `MobileInkCanvasView` is the Metal-backed native drawing surface. +- Android `MobileInkCanvasView` is a `GLSurfaceView` that renders the shared C++ engine into an OpenGL texture. +- `MobileInkCanvasViewManager` exposes React Native props and commands on both platforms. +- `MobileInkBridge` exposes iOS helpers that are not tied to a single view, including notebook parsing and continuous-window compose/decompose. +- `MobileInkModule` exposes Android promise-based drawing persistence and batch export helpers. +- `MobileInkBackgroundView` renders generic page backgrounds on iOS; Android backgrounds are rendered by the shared Skia engine inside the canvas. ### C++ Skia engine @@ -57,4 +59,4 @@ Consumers own storage and decide how serialized notebook payloads move through t ## Native Memory Lifecycle -The pool reuses native views for page changes. Heavy native state is released only on final unmount through `releaseEngine`, which prevents scroll-driven MTKView churn and keeps allocations flat during repeated page crossing. +The pool reuses native views for page changes. Heavy native state is released only on final unmount through `releaseEngine` where the platform needs explicit teardown, which prevents scroll-driven native-view churn and keeps allocations flat during repeated page crossing. diff --git a/example/App.tsx b/example/App.tsx index 4e6ca7a..826d67a 100644 --- a/example/App.tsx +++ b/example/App.tsx @@ -1,5 +1,6 @@ -import React, { useEffect, useRef, useState } from "react"; +import React, { useCallback, useEffect, useRef, useState } from "react"; import { + Platform, SafeAreaView, StyleSheet, Switch, @@ -23,9 +24,15 @@ import { BenchmarkScreen } from "./BenchmarkScreen"; type AppMode = "draw" | "benchmark"; const STORAGE_KEY = "mobile-ink-example-notebook"; +const supportsNativeBenchmark = Platform.OS === "ios"; +const supportsRenderBackendToggle = Platform.OS === "ios"; export default function App() { const canvasRef = useRef(null); + const autosaveTimerRef = useRef | null>(null); + const canvasReadyRef = useRef(false); + const hasUserEditedRef = useRef(false); + const pendingNotebookRef = useRef(null); const [mode, setMode] = useState("draw"); const [activeTool, setActiveTool] = useState(tools[0]); const [drawWithFinger, setDrawWithFinger] = useState(false); @@ -36,6 +43,21 @@ export default function App() { const [pageCount, setPageCount] = useState(1); const [isMoving, setIsMoving] = useState(false); + const loadNotebook = useCallback(async (notebookData: SerializedNotebookData) => { + await canvasRef.current?.loadNotebookData(notebookData); + hasUserEditedRef.current = false; + setStorageStatus("saved on disk"); + }, []); + + useEffect(() => { + return () => { + if (autosaveTimerRef.current) { + clearTimeout(autosaveTimerRef.current); + autosaveTimerRef.current = null; + } + }; + }, []); + useEffect(() => { let isMounted = true; AsyncStorage.getItem(STORAGE_KEY) @@ -44,8 +66,15 @@ export default function App() { return; } - setSavedNotebook(JSON.parse(rawNotebook) as SerializedNotebookData); + const notebookData = JSON.parse(rawNotebook) as SerializedNotebookData; + setSavedNotebook(notebookData); setStorageStatus("saved on disk"); + + if (canvasReadyRef.current && !hasUserEditedRef.current) { + void loadNotebook(notebookData); + } else { + pendingNotebookRef.current = notebookData; + } }) .catch(() => { if (isMounted) { @@ -56,36 +85,68 @@ export default function App() { return () => { isMounted = false; }; - }, []); + }, [loadNotebook]); const applyTool = (tool: ToolConfig) => { setActiveTool(tool); canvasRef.current?.setTool(tool); }; - const save = async () => { - const notebookData = await canvasRef.current?.getNotebookData(); - if (!notebookData) { - return; + const save = useCallback(async () => { + try { + setStorageStatus("saving..."); + const notebookData = await canvasRef.current?.getNotebookData(); + if (!notebookData) { + setStorageStatus("save unavailable"); + return; + } + + await AsyncStorage.setItem(STORAGE_KEY, JSON.stringify(notebookData)); + setSavedNotebook(notebookData); + setStorageStatus("saved on disk"); + } catch { + setStorageStatus("save failed"); } + }, []); - await AsyncStorage.setItem(STORAGE_KEY, JSON.stringify(notebookData)); - setSavedNotebook(notebookData); - setStorageStatus("saved on disk"); - }; + const scheduleAutosave = useCallback(() => { + hasUserEditedRef.current = true; + setStorageStatus("unsaved changes"); + if (autosaveTimerRef.current) { + clearTimeout(autosaveTimerRef.current); + } + + autosaveTimerRef.current = setTimeout(() => { + autosaveTimerRef.current = null; + void save(); + }, 1000); + }, [save]); const reload = async () => { const notebookData = savedNotebook; if (notebookData) { - await canvasRef.current?.loadNotebookData(notebookData); + if (autosaveTimerRef.current) { + clearTimeout(autosaveTimerRef.current); + autosaveTimerRef.current = null; + } + await loadNotebook(notebookData); } }; + const handleCanvasReady = useCallback(() => { + canvasReadyRef.current = true; + const pendingNotebook = pendingNotebookRef.current; + if (pendingNotebook && !hasUserEditedRef.current) { + pendingNotebookRef.current = null; + void loadNotebook(pendingNotebook); + } + }, [loadNotebook]); + return ( - {(["draw", "benchmark"] as AppMode[]).map((item) => ( + {((supportsNativeBenchmark ? ["draw", "benchmark"] : ["draw"]) as AppMode[]).map((item) => ( - - {(["ganesh", "cpu"] as NativeInkRenderBackend[]).map((item) => ( - setRenderBackend(item)} - > - {item === "ganesh" ? "Ganesh" : "CPU"} - - ))} - + {supportsRenderBackendToggle ? ( + + {(["ganesh", "cpu"] as NativeInkRenderBackend[]).map((item) => ( + setRenderBackend(item)} + > + {item === "ganesh" ? "Ganesh" : "CPU"} + + ))} + + ) : null} Draw with finger @@ -152,7 +215,9 @@ export default function App() { Page {currentPageIndex + 1} / {pageCount} {isMoving ? "moving" : "settled"} {storageStatus} - Backend {renderBackend} + {supportsRenderBackendToggle ? ( + Backend {renderBackend} + ) : null} ) : null} @@ -169,6 +234,8 @@ export default function App() { toolState={activeTool} minScale={1} maxScale={5} + onReady={handleCanvasReady} + onDrawingChange={scheduleAutosave} onCurrentPageChange={setCurrentPageIndex} onPagesChange={(pages) => setPageCount(pages.length)} onMotionStateChange={setIsMoving} diff --git a/example/README.md b/example/README.md new file mode 100644 index 0000000..a7871da --- /dev/null +++ b/example/README.md @@ -0,0 +1,63 @@ +# Mobile Ink Example + +The example is an Expo dev-client app that loads `@mathnotes/mobile-ink` from the parent folder with `file:..`. It cannot run in Expo Go because the package includes Kotlin, C++, and platform native views. + +## Android + +Prerequisites: + +- Android Studio with an installed SDK, platform tools, emulator, and NDK/CMake support. +- A running Android emulator or a USB device with developer mode enabled. +- `JAVA_HOME` and `ANDROID_HOME` configured for your Android toolchain. + +From the repository root: + +```sh +npm ci +npm ci --prefix example +cd example +npx expo run:android +``` + +The first Android run builds the native library, including the shared C++ drawing engine. That build can take a while and should include Gradle tasks such as `:mathnotes_mobile-ink:compileDebugKotlin` and `:mathnotes_mobile-ink:externalNativeBuildDebug`. + +If Metro is already running, use the dev-client command in a second terminal: + +```sh +cd example +npx expo start --dev-client +``` + +If native code or Android package metadata changed and the generated native project looks stale, rebuild it from Expo config: + +```sh +cd example +npx expo prebuild --platform android --clean +npx expo run:android +``` + +The generated `example/android/` folder is local build output and is intentionally ignored by git. + +## Android Smoke Checks + +These checks do not require a connected device: + +```sh +npm --prefix example run typecheck +npm --prefix example run export:android +``` + +If the installed app reports a missing native module at runtime, rebuild the dev client with `npx expo run:android`. Static export validates Metro bundling; it does not compile or install native code. + +## iOS + +```sh +npm ci +npm ci --prefix example +cd example +npx expo run:ios +``` + +Use `npx expo run:ios --device` for a physical device. + +The benchmark screen and render-backend toggle are iOS-only in this example because the benchmark recorder and CPU/Ganesh backend selector are currently implemented on the iOS native bridge. diff --git a/example/package-lock.json b/example/package-lock.json index ae295fd..502c733 100644 --- a/example/package-lock.json +++ b/example/package-lock.json @@ -6393,9 +6393,9 @@ } }, "node_modules/postcss": { - "version": "8.4.49", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.49.tgz", - "integrity": "sha512-OCVPnIObs4N29kxTjzLfUryOkvZEq+pf8jTF0lg8E7uETuWHA+v7j3c/xJmiqpX450191LlmZfUKkXxkTry7nA==", + "version": "8.5.10", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.10.tgz", + "integrity": "sha512-pMMHxBOZKFU6HgAZ4eyGnwXF/EvPGGqUr0MnZ5+99485wwW41kW91A4LOGxSHhgugZmSChL5AlElNdwlNgcnLQ==", "funding": [ { "type": "opencollective", @@ -6412,7 +6412,7 @@ ], "license": "MIT", "dependencies": { - "nanoid": "^3.3.7", + "nanoid": "^3.3.11", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" }, diff --git a/example/package.json b/example/package.json index a6421ca..7eefced 100644 --- a/example/package.json +++ b/example/package.json @@ -8,7 +8,8 @@ "ios": "expo run:ios", "android": "expo run:android", "typecheck": "tsc --noEmit -p tsconfig.json", - "export:ios": "expo export --platform ios --output-dir /tmp/mobile-ink-example-export" + "export:ios": "expo export --platform ios --output-dir /tmp/mobile-ink-example-export-ios", + "export:android": "expo export --platform android --output-dir /tmp/mobile-ink-example-export-android" }, "dependencies": { "@mathnotes/mobile-ink": "file:..", @@ -27,5 +28,8 @@ "babel-preset-expo": "54.0.10", "typescript": "~5.9.2", "@types/react": "^19.1.0" + }, + "overrides": { + "postcss": "8.5.10" } } diff --git a/ios/MobileInkModule/MobileInkCanvasView.swift b/ios/MobileInkModule/MobileInkCanvasView.swift index 1ab2670..2b58d09 100644 --- a/ios/MobileInkModule/MobileInkCanvasView.swift +++ b/ios/MobileInkModule/MobileInkCanvasView.swift @@ -138,7 +138,7 @@ class MobileInkCanvasView: MTKView { @objc var onDrawingChange: RCTDirectEventBlock? @objc var onDrawingBegin: RCTDirectEventBlock? - @objc var onSelectionChange: RCTDirectEventBlock? + @objc var onInkSelectionChange: RCTDirectEventBlock? @objc var onPencilDoubleTap: RCTDirectEventBlock? private var pencilDoubleTapSequence: Int = 0 private var pendingPresentedLoadCallbacks: [RCTResponseSenderBlock] = [] @@ -273,7 +273,7 @@ class MobileInkCanvasView: MTKView { hasDeferredPresentation = false onDrawingChange = nil onDrawingBegin = nil - onSelectionChange = nil + onInkSelectionChange = nil onPencilDoubleTap = nil delegate = nil commandQueue = nil @@ -1427,7 +1427,7 @@ class MobileInkCanvasView: MTKView { payload["bounds"] = NSNull() } updateSelectionToolbarFrame() - onSelectionChange?(payload) + onInkSelectionChange?(payload) } private func serializedDrawingData() -> Data? { diff --git a/ios/MobileInkModule/MobileInkCanvasViewManager.m b/ios/MobileInkModule/MobileInkCanvasViewManager.m index c0764b4..4ef55ef 100644 --- a/ios/MobileInkModule/MobileInkCanvasViewManager.m +++ b/ios/MobileInkModule/MobileInkCanvasViewManager.m @@ -5,7 +5,7 @@ @interface RCT_EXTERN_MODULE(MobileInkCanvasViewManager, RCTViewManager) RCT_EXPORT_VIEW_PROPERTY(onDrawingChange, RCTDirectEventBlock) RCT_EXPORT_VIEW_PROPERTY(onDrawingBegin, RCTDirectEventBlock) -RCT_EXPORT_VIEW_PROPERTY(onSelectionChange, RCTDirectEventBlock) +RCT_EXPORT_VIEW_PROPERTY(onInkSelectionChange, RCTDirectEventBlock) RCT_EXPORT_VIEW_PROPERTY(onPencilDoubleTap, RCTDirectEventBlock) RCT_EXPORT_VIEW_PROPERTY(backgroundType, NSString) RCT_EXPORT_VIEW_PROPERTY(pdfBackgroundUri, NSString) diff --git a/ios/MobileInkModule/MobileInkCanvasViewManager.swift b/ios/MobileInkModule/MobileInkCanvasViewManager.swift index 4bc5c7d..f540d28 100644 --- a/ios/MobileInkModule/MobileInkCanvasViewManager.swift +++ b/ios/MobileInkModule/MobileInkCanvasViewManager.swift @@ -40,9 +40,9 @@ class DrawingContainerView: UIView, UIPencilInteractionDelegate { } } - @objc var onSelectionChange: RCTDirectEventBlock? { + @objc var onInkSelectionChange: RCTDirectEventBlock? { didSet { - drawingView?.onSelectionChange = onSelectionChange + drawingView?.onInkSelectionChange = onInkSelectionChange } } @@ -107,8 +107,8 @@ class DrawingContainerView: UIView, UIPencilInteractionDelegate { if let handler = onDrawingBegin { view.onDrawingBegin = handler } - if let handler = onSelectionChange { - view.onSelectionChange = handler + if let handler = onInkSelectionChange { + view.onInkSelectionChange = handler } if let handler = onPencilDoubleTap { view.onPencilDoubleTap = handler diff --git a/package.json b/package.json index d9c71c5..0bdf019 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@mathnotes/mobile-ink", "version": "0.2.0", - "description": "Production-grade React Native ink engine with native Skia/Metal drawing and continuous canvas primitives.", + "description": "Production-grade React Native ink engine with native Skia drawing and continuous canvas primitives.", "license": "Apache-2.0", "author": "BuilderPro LLC", "private": false, @@ -25,6 +25,7 @@ "src/benchmark", "src/utils", "ios", + "android", "cpp", "docs/*.md", "CHANGELOG.md", @@ -41,8 +42,9 @@ "test:native:smoke": "clang++ -std=c++20 scripts/drawing_serialization_smoke.cpp cpp/DrawingTypes.cpp cpp/DrawingSerialization.cpp cpp/ShapeRecognition.cpp -I cpp -I node_modules/@shopify/react-native-skia/cpp/skia -I node_modules/@shopify/react-native-skia/cpp/skia/modules/pathops/include node_modules/@shopify/react-native-skia/libs/apple/libskia.xcframework/macos-arm64_x86_64/libskia.a node_modules/@shopify/react-native-skia/libs/apple/libpathops.xcframework/macos-arm64_x86_64/libpathops.a -framework ApplicationServices -framework CoreFoundation -framework CoreGraphics -framework CoreText -framework Foundation -framework QuartzCore -o /tmp/mobile_ink_drawing_serialization_smoke && /tmp/mobile_ink_drawing_serialization_smoke", "test:example:typecheck": "npm --prefix example run typecheck", "test:example:export:ios": "npm --prefix example run export:ios", + "test:example:export:android": "npm --prefix example run export:android", "pack:dry-run": "npm pack --dry-run", - "validate": "npm run typecheck && npm run test && npm run test:native:smoke && npm run build && npm run pack:dry-run && npm run test:example:typecheck && npm run test:example:export:ios", + "validate": "npm run typecheck && npm run test && npm run test:native:smoke && npm run build && npm run pack:dry-run && npm run test:example:typecheck && npm run test:example:export:ios && npm run test:example:export:android", "prepack": "npm run build" }, "peerDependencies": { diff --git a/react-native.config.js b/react-native.config.js index 47789b5..bdf4dfa 100644 --- a/react-native.config.js +++ b/react-native.config.js @@ -4,7 +4,11 @@ module.exports = { ios: { podspecPath: 'MathNotesMobileInk.podspec', }, - android: null, + android: { + sourceDir: 'android', + packageImportPath: 'import com.mathnotes.mobileink.MobileInkPackage;', + packageInstance: 'new MobileInkPackage()', + }, }, }, }; diff --git a/src/InfiniteInkCanvas.tsx b/src/InfiniteInkCanvas.tsx index 651ec11..1883bcb 100644 --- a/src/InfiniteInkCanvas.tsx +++ b/src/InfiniteInkCanvas.tsx @@ -43,6 +43,7 @@ import type { InfiniteInkViewportTransform, } from "./infinite-ink-canvas/types"; import { getContinuousEnginePoolRange } from "./utils/continuousEnginePool"; +import { computeDataSignature } from "./utils/dataSignature"; import { BLANK_PAGE_PAYLOAD, withSingleTrailingBlankPage, @@ -54,6 +55,32 @@ export type { InfiniteInkViewportTransform, } from "./infinite-ink-canvas/types"; +const PAGE_PREVIEW_CAPTURE_SCALE = 0.25; + +const withCapturedPageData = ( + page: NotebookPage, + data: string, + previewUri?: string, +): NotebookPage => { + const dataSignature = computeDataSignature(data); + const hasFreshPreview = Boolean(previewUri); + const hasMatchingPreview = page.previewDataSignature === dataSignature; + + return { + ...page, + data, + dataSignature, + previewUri: hasFreshPreview + ? previewUri + : hasMatchingPreview + ? page.previewUri + : undefined, + previewDataSignature: hasFreshPreview || hasMatchingPreview + ? dataSignature + : undefined, + }; +}; + function InfiniteInkCanvasImpl( { style, @@ -220,13 +247,17 @@ function InfiniteInkCanvasImpl( return dirtyPageIdsRef.current.has(pageId); }, []); - const updatePageData = useCallback((pageId: string, data: string) => { + const updatePageData = useCallback(( + pageId: string, + data: string, + previewUri?: string, + ) => { if (!data) { return; } const nextPages = pagesRef.current.map((page) => ( - page.id === pageId ? { ...page, data } : page + page.id === pageId ? withCapturedPageData(page, data, previewUri) : page )); dirtyPageIdsRef.current.delete(pageId); replacePages(withSingleTrailingBlankPage(nextPages, dirtyPageIdsRef.current)); @@ -276,12 +307,17 @@ function InfiniteInkCanvasImpl( } const data = await slotRef.getBase64Data(); + const previewUri = await slotRef.getPreviewData(PAGE_PREVIEW_CAPTURE_SCALE); const pageIndex = nextPages.findIndex((page) => page.id === pageId); if (pageIndex === -1 || !data) { continue; } - nextPages[pageIndex] = { ...nextPages[pageIndex], data }; + nextPages[pageIndex] = withCapturedPageData( + nextPages[pageIndex], + data, + previewUri ?? undefined, + ); dirtyPageIdsRef.current.delete(pageId); didChange = true; } @@ -361,7 +397,13 @@ function InfiniteInkCanvasImpl( getActiveSlot()?.clear(); const nextPages = pagesRef.current.map((candidatePage) => ( candidatePage.id === page.id - ? { ...candidatePage, data: BLANK_PAGE_PAYLOAD } + ? { + ...candidatePage, + data: BLANK_PAGE_PAYLOAD, + dataSignature: computeDataSignature(BLANK_PAGE_PAYLOAD), + previewUri: undefined, + previewDataSignature: undefined, + } : candidatePage )); replacePages(withSingleTrailingBlankPage(nextPages, dirtyPageIdsRef.current)); diff --git a/src/ZoomableInkViewport.tsx b/src/ZoomableInkViewport.tsx index d58ded7..8120b57 100644 --- a/src/ZoomableInkViewport.tsx +++ b/src/ZoomableInkViewport.tsx @@ -220,6 +220,10 @@ const ZoomableInkViewport = forwardRef ({ + height: Math.max(0, contentHeight + contentPadding * 2), + }), [contentHeight, contentPadding]); + const isZoomed = useCallback(() => { return scale.value > 1.05; }, [scale]); @@ -272,6 +276,11 @@ const ZoomableInkViewport = forwardRef { return { + transformOrigin: [ + containerWidth.value / 2, + containerHeight.value / 2, + 0, + ], transform: [ { translateX: translateX.value }, { translateY: translateY.value }, @@ -284,7 +293,9 @@ const ZoomableInkViewport = forwardRef - {children} + + {children} + ); } @@ -296,7 +307,10 @@ const ZoomableInkViewport = forwardRef - + {children} @@ -315,7 +329,10 @@ const styles = StyleSheet.create({ flex: 1, }, transformedContent: { - ...StyleSheet.absoluteFillObject, + position: 'absolute', + top: 0, + left: 0, + right: 0, }, }); diff --git a/src/__tests__/ContinuousEnginePool.test.tsx b/src/__tests__/ContinuousEnginePool.test.tsx index c350808..a6ff952 100644 --- a/src/__tests__/ContinuousEnginePool.test.tsx +++ b/src/__tests__/ContinuousEnginePool.test.tsx @@ -1,4 +1,5 @@ import React from "react"; +import { StyleSheet } from "react-native"; import { act, render } from "@testing-library/react-native"; import { ContinuousEnginePool, @@ -11,6 +12,7 @@ import type { NotebookPage } from "../types"; const mockLoadBase64Data = jest.fn(async () => true); const mockGetBase64Data = jest.fn(async () => '{"pages":{"0":"persisted"}}'); +const mockGetBase64PngData = jest.fn(async () => "data:image/png;base64,preview"); const mockReleaseEngine = jest.fn(async () => undefined); const mockSetTool = jest.fn(); const mockSetNativeProps = jest.fn(); @@ -63,6 +65,7 @@ jest.mock("../NativeInkCanvas", () => { setNativeProps: mockSetNativeProps, loadBase64Data: mockLoadBase64Data, getBase64Data: mockGetBase64Data, + getBase64PngData: mockGetBase64PngData, releaseEngine: mockReleaseEngine, runBenchmark: mockRunBenchmark, startBenchmarkRecording: mockStartBenchmarkRecording, @@ -173,6 +176,7 @@ describe("ContinuousEnginePool", () => { expect(onSlotCaptureBeforeUnmount).toHaveBeenCalledWith( "page-0", '{"pages":{"0":"persisted"}}', + "data:image/png;base64,preview", ); view.unmount(); @@ -239,6 +243,23 @@ describe("ContinuousEnginePool", () => { expect(mockLoadBase64Data).not.toHaveBeenCalled(); }); + it("retries page loads until the native engine accepts the payload", async () => { + const pages = [page(0)]; + mockLoadBase64Data + .mockResolvedValueOnce(false) + .mockResolvedValueOnce(false) + .mockResolvedValueOnce(true); + const { poolRef } = renderPool(); + + await act(async () => {}); + await assignPages(poolRef, buildAssignments(pages, 0)); + + expect(mockLoadBase64Data).toHaveBeenCalledTimes(3); + expect(mockLoadBase64Data).toHaveBeenNthCalledWith(1, "data-0"); + expect(mockLoadBase64Data).toHaveBeenNthCalledWith(2, "data-0"); + expect(mockLoadBase64Data).toHaveBeenNthCalledWith(3, "data-0"); + }); + it("preserves already loaded page slots across adjacent page changes", async () => { const pages = [page(0), page(1), page(2), page(3)]; const { poolRef } = renderPool(); @@ -253,6 +274,33 @@ describe("ContinuousEnginePool", () => { expect(mockLoadBase64Data).toHaveBeenCalledWith("data-3"); }); + it("keeps pooled slot frames in React layout state", async () => { + const pages = [page(0), page(1), page(2)]; + const { poolRef, view } = renderPool(); + + await act(async () => {}); + await assignPages(poolRef, buildAssignments(pages, 0)); + + const firstSlot = view.getByTestId("continuous-engine-pool-slot-0"); + const secondSlot = view.getByTestId("continuous-engine-pool-slot-1"); + expect(firstSlot.props.pointerEvents).toBe("auto"); + expect(StyleSheet.flatten(firstSlot.props.style)).toEqual( + expect.objectContaining({ top: 0, height: 800, opacity: 1 }), + ); + expect(secondSlot.props.pointerEvents).toBe("auto"); + expect(StyleSheet.flatten(secondSlot.props.style)).toEqual( + expect.objectContaining({ top: 800, height: 800, opacity: 1 }), + ); + + await assignPages(poolRef, buildAssignments([pages[0]], 0)); + + const hiddenSecondSlot = view.getByTestId("continuous-engine-pool-slot-1"); + expect(hiddenSecondSlot.props.pointerEvents).toBe("none"); + expect(StyleSheet.flatten(hiddenSecondSlot.props.style)).toEqual( + expect.objectContaining({ top: -100000, height: 800, opacity: 0 }), + ); + }); + it("forwards pencil double-tap events to every pooled native canvas", async () => { const onPencilDoubleTap = jest.fn(); diff --git a/src/__tests__/serialization.test.ts b/src/__tests__/serialization.test.ts index d76c71a..82bc3c7 100644 --- a/src/__tests__/serialization.test.ts +++ b/src/__tests__/serialization.test.ts @@ -319,6 +319,26 @@ describe('serialization round-trip', () => { expect(deserializedPages[1].pdfPageNumber).toBe(2); expect(deserializedPages[2].pdfPageNumber).toBe(3); }); + + it('preserves page preview metadata through round-trip', () => { + const originalPages: NotebookPage[] = [ + { + id: 'page-1', + title: 'Page 1', + rotation: 0, + data: 'drawing', + dataSignature: '7:drawing', + previewUri: 'data:image/png;base64,preview', + previewDataSignature: '7:drawing', + }, + ]; + + const serialized = serializeNotebookData(originalPages); + const { pages: deserializedPages } = deserializeNotebookData(serialized); + + expect(deserializedPages[0].previewUri).toBe('data:image/png;base64,preview'); + expect(deserializedPages[0].previewDataSignature).toBe('7:drawing'); + }); }); describe('data integrity edge cases', () => { diff --git a/src/continuous-engine-pool/PooledCanvasSlot.tsx b/src/continuous-engine-pool/PooledCanvasSlot.tsx index 0300c3d..d3d712f 100644 --- a/src/continuous-engine-pool/PooledCanvasSlot.tsx +++ b/src/continuous-engine-pool/PooledCanvasSlot.tsx @@ -6,14 +6,16 @@ import React, { useImperativeHandle, useMemo, useRef, + useState, } from "react"; -import { StyleSheet, View } from "react-native"; +import { Image, StyleSheet, View } from "react-native"; import { NativeInkCanvas } from "../NativeInkCanvas"; import type { NativeInkBenchmarkRecordingOptions } from "../benchmark"; import type { NativeSelectionBounds } from "../types"; import { BLANK_PAGE_PAYLOAD, getPdfBackgroundUri, + loadCanvasDataWithRetry, OFFSCREEN_TOP, waitForNextFrame, } from "./helpers"; @@ -27,6 +29,24 @@ import type { PooledCanvasSlotProps, } from "./types"; +type SlotPointerEvents = "auto" | "none"; + +type SlotFrame = { + top: number; + height: number; + opacity: number; + pointerEvents: SlotPointerEvents; +}; + +const PAGE_PREVIEW_CAPTURE_SCALE = 0.25; + +const hiddenSlotFrame = (height = 0): SlotFrame => ({ + top: OFFSCREEN_TOP, + height, + opacity: 0, + pointerEvents: "none", +}); + export const PooledCanvasSlot = memo(forwardRef( function PooledCanvasSlot({ poolIndex, @@ -48,6 +68,9 @@ export const PooledCanvasSlot = memo(forwardRef(null); const canvasRef = useRef(null); + const [slotFrame, setSlotFrameState] = useState(() => hiddenSlotFrame()); + const [previewUri, setPreviewUri] = useState(null); + const [isPreviewVisible, setIsPreviewVisible] = useState(false); const lastAttachedCanvasRef = useRef(null); const nativeReadyRef = useRef(false); const nativeReadyWaitersRef = useRef void>>([]); @@ -93,12 +116,23 @@ export const PooledCanvasSlot = memo(forwardRef { + const nextFrame: SlotFrame = pageIndex === null + ? hiddenSlotFrame(height) + : { + top: pageIndex * height, + height, + opacity: isVisible ? 1 : 0, + pointerEvents: isInteractive ? "auto" : "none", + }; + + setSlotFrameState(nextFrame); slotViewRef.current?.setNativeProps({ style: { - top: pageIndex === null ? OFFSCREEN_TOP : pageIndex * height, - height, - opacity: pageIndex === null || !isVisible ? 0 : 1, + top: nextFrame.top, + height: nextFrame.height, + opacity: nextFrame.opacity, }, }); }, []); @@ -135,6 +169,14 @@ export const PooledCanvasSlot = memo(forwardRef { + if (!canvasRef.current) return null; + try { + return await canvasRef.current.getBase64PngData(scale); + } catch { + return null; + } + }, isLoaded: () => isLoadedRef.current, setTool: (toolType, width, color, eraserMode) => { canvasRef.current?.setTool(toolType, width, color, eraserMode); @@ -195,7 +237,13 @@ export const PooledCanvasSlot = memo(forwardRef + {previewUri ? ( + + ) : null} new Promise((resolve) => { requestAnimationFrame(() => resolve()); }); + +export const loadCanvasDataWithRetry = async ( + canvas: NativeCanvasRef, + payload: string, + attempts = 90, +) => { + for (let attempt = 0; attempt < attempts; attempt += 1) { + if (await canvas.loadBase64Data(payload)) { + return true; + } + + await waitForNextFrame(); + } + + return false; +}; diff --git a/src/continuous-engine-pool/types.ts b/src/continuous-engine-pool/types.ts index fad99f6..65ef308 100644 --- a/src/continuous-engine-pool/types.ts +++ b/src/continuous-engine-pool/types.ts @@ -24,6 +24,7 @@ export type ContinuousEnginePoolToolState = { export type ContinuousEnginePoolSlotRef = { getBase64Data: () => Promise; + getPreviewData: (scale?: number) => Promise; isLoaded: () => boolean; setTool: ( toolType: string, @@ -78,7 +79,11 @@ export type ContinuousEnginePoolProps = { sourceRef?: ContinuousEnginePoolSlotRef, ) => void; shouldCaptureBeforeReassign: (pageId: string) => boolean; - onSlotCaptureBeforeUnmount: (pageId: string, data: string) => void; + onSlotCaptureBeforeUnmount: ( + pageId: string, + data: string, + previewUri?: string, + ) => void; }; export type PooledCanvasSlotAssignOptions = { @@ -126,7 +131,11 @@ export type PooledCanvasSlotProps = { sourceRef?: ContinuousEnginePoolSlotRef, ) => void; shouldCaptureBeforeReassign: (pageId: string) => boolean; - onCaptureBeforeReassign: (pageId: string, data: string) => void; + onCaptureBeforeReassign: ( + pageId: string, + data: string, + previewUri?: string, + ) => void; }; export type NativeCanvasRef = NativeInkCanvasRef; diff --git a/src/native-ink-canvas/notebookBridge.ts b/src/native-ink-canvas/notebookBridge.ts index a7e8530..e81d865 100644 --- a/src/native-ink-canvas/notebookBridge.ts +++ b/src/native-ink-canvas/notebookBridge.ts @@ -72,7 +72,8 @@ export async function batchExportPages( width, height, scale, - pdfBackgroundUri || "" + pdfBackgroundUri || "", + pageIndices || [] ); } @@ -98,17 +99,22 @@ export async function batchExportPages( * or if the native fast path isn't available (older build). Rejects on real * read/parse errors so the caller can fall back to the slow path. * - * iOS-only: MobileInkBridge ships the parser. Android falls through to - * the existing JS-side read+parse path. + * MobileInkBridge/MobileInkModule ship the parser on native platforms. */ export async function readBodyFileParsed( bodyPath: string, ): Promise | null> { - if (Platform.OS !== "ios" || !MobileInkBridge?.readBodyFileParsed) { + if ( + (Platform.OS === "ios" && !MobileInkBridge?.readBodyFileParsed) || + (Platform.OS === "android" && !MobileInkModule?.readBodyFileParsed) || + (Platform.OS !== "ios" && Platform.OS !== "android") + ) { return null; } try { - const result = await MobileInkBridge.readBodyFileParsed(bodyPath); + const result = Platform.OS === "ios" + ? await MobileInkBridge.readBodyFileParsed(bodyPath) + : await MobileInkModule.readBodyFileParsed(bodyPath); if (result === null || result === undefined) return null; if (typeof result !== "object") return null; return result as Record; @@ -125,11 +131,14 @@ export async function composeContinuousWindow( pagePayloads: string[], pageHeight: number ): Promise { - if (Platform.OS !== "ios") { - throw new Error("Continuous window composition is only available on iOS."); + if (Platform.OS === "android") { + if (!MobileInkModule?.composeContinuousWindow) { + throw new Error("MobileInkModule.composeContinuousWindow not found. Please rebuild the app."); + } + return MobileInkModule.composeContinuousWindow(pagePayloads, pageHeight); } - if (!MobileInkBridge?.composeContinuousWindow) { + if (Platform.OS !== "ios" || !MobileInkBridge?.composeContinuousWindow) { throw new Error("MobileInkBridge.composeContinuousWindow not found. Please rebuild the app."); } @@ -141,11 +150,14 @@ export async function decomposeContinuousWindow( pageCount: number, pageHeight: number ): Promise { - if (Platform.OS !== "ios") { - throw new Error("Continuous window decomposition is only available on iOS."); + if (Platform.OS === "android") { + if (!MobileInkModule?.decomposeContinuousWindow) { + throw new Error("MobileInkModule.decomposeContinuousWindow not found. Please rebuild the app."); + } + return MobileInkModule.decomposeContinuousWindow(windowPayload, pageCount, pageHeight); } - if (!MobileInkBridge?.decomposeContinuousWindow) { + if (Platform.OS !== "ios" || !MobileInkBridge?.decomposeContinuousWindow) { throw new Error("MobileInkBridge.decomposeContinuousWindow not found. Please rebuild the app."); } diff --git a/src/types.ts b/src/types.ts index 6d866d5..d905b38 100644 --- a/src/types.ts +++ b/src/types.ts @@ -86,6 +86,8 @@ export type NotebookPage = { title: string; data?: string; dataSignature?: string; + previewUri?: string; + previewDataSignature?: string; rotation: number; textBoxes?: InkTextBox[]; insertedElements?: InsertedElement[]; diff --git a/src/utils/dataSignature.ts b/src/utils/dataSignature.ts new file mode 100644 index 0000000..0a62c8a --- /dev/null +++ b/src/utils/dataSignature.ts @@ -0,0 +1,5 @@ +export const computeDataSignature = (data: string): string => { + const len = data.length; + if (len <= 128) return `${len}:${data}`; + return `${len}:${data.slice(0, 64)}:${data.slice(-64)}`; +}; diff --git a/src/utils/serialization.ts b/src/utils/serialization.ts index 53cf0f7..200a40a 100644 --- a/src/utils/serialization.ts +++ b/src/utils/serialization.ts @@ -1,4 +1,5 @@ import { NotebookPage, SerializedNotebookData, TextBox } from '../types'; +import { computeDataSignature } from './dataSignature'; /** * Serialization utilities for notebook data persistence. @@ -17,6 +18,9 @@ export const serializeNotebookData = (pages: NotebookPage[], canvasWidth?: numbe id: p.id, title: p.title, data: p.data, + dataSignature: p.dataSignature, + previewUri: p.previewUri, + previewDataSignature: p.previewDataSignature, rotation: p.rotation, textBoxes: p.textBoxes, insertedElements: p.insertedElements, @@ -59,17 +63,6 @@ function createBlankPdfPages(startIndex: number, pageCount: number): NotebookPag * path (readBodyFileParsed -> structured object) can share the same * migration logic without paying a JSON.parse round-trip on the JS side. */ -// Stable content fingerprint for a page's base64 body. Used by the -// preview-cache machinery to detect real content changes vs. mere -// in-memory eviction. Same shape as computePayloadSignature in -// useContinuousPageManagement -- duplicated here so the pure -// serialization layer doesn't need to import that hook. -const computeDataSignature = (data: string): string => { - const len = data.length; - if (len <= 128) return `${len}:${data}`; - return `${len}:${data.slice(0, 64)}:${data.slice(-64)}`; -}; - export const buildNotebookFromParsed = ( data: SerializedNotebookData | null | undefined, ): DeserializedNotebookData => {