diff --git a/androidApp/src/main/kotlin/dev/stapler/stelekit/SteleKitApplication.kt b/androidApp/src/main/kotlin/dev/stapler/stelekit/SteleKitApplication.kt index 24512671..83ac3801 100644 --- a/androidApp/src/main/kotlin/dev/stapler/stelekit/SteleKitApplication.kt +++ b/androidApp/src/main/kotlin/dev/stapler/stelekit/SteleKitApplication.kt @@ -9,6 +9,11 @@ import dev.stapler.stelekit.platform.PlatformFileSystem import dev.stapler.stelekit.platform.PlatformSettings import dev.stapler.stelekit.platform.SteleKitContext import dev.stapler.stelekit.platform.WriteBehindQueue +import dev.stapler.stelekit.platform.measurement.MeasurementDeviceRegistry +import dev.stapler.stelekit.platform.measurement.ble.KableBleScanner +import dev.stapler.stelekit.platform.sensor.AndroidCameraProvider +import dev.stapler.stelekit.platform.sensor.ARCoreDepthProvider +import dev.stapler.stelekit.platform.sensor.SensorModule import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.SupervisorJob @@ -41,6 +46,9 @@ class SteleKitApplication : Application() { SteleKitContext.init(this) DriverFactory.setContext(this) CredentialStore.init(this) + SensorModule.cameraProvider = AndroidCameraProvider(applicationContext) + SensorModule.depthSensorProvider = ARCoreDepthProvider(applicationContext) + MeasurementDeviceRegistry.register(KableBleScanner(applicationContext)) fileSystem = PlatformFileSystem().apply { init(applicationContext) } // Activate write-behind when MANAGE_EXTERNAL_STORAGE is not granted. // Direct access (when granted) is faster than write-behind and makes it unnecessary. diff --git a/kmp/build.gradle.kts b/kmp/build.gradle.kts index 7f7db3e3..8aed8e59 100644 --- a/kmp/build.gradle.kts +++ b/kmp/build.gradle.kts @@ -196,6 +196,21 @@ kotlin { // Encrypted SharedPreferences for API key storage implementation("androidx.security:security-crypto:1.1.0-alpha06") + // ExifInterface — EXIF orientation correction for camera-captured images + implementation("androidx.exifinterface:exifinterface:1.3.7") + + // ARCore Depth API — optional AR depth sensing (Story 8.5) + // required=false in AndroidManifest so the app installs on non-AR devices. + implementation("com.google.ar:core:1.46.0") + + // CameraX — live camera capture (Story 9.2) + implementation("androidx.camera:camera-core:1.4.1") + implementation("androidx.camera:camera-camera2:1.4.1") + implementation("androidx.camera:camera-lifecycle:1.4.1") + // ProcessLifecycleOwner — needed by AndroidCameraProvider; camera-lifecycle pulls + // this transitively but we declare it explicitly to guarantee compile-time resolution. + implementation("androidx.lifecycle:lifecycle-process:2.9.1") + // On-device LLM via Gemini Nano (Pixel 9+ and AICore-enabled OEM flagships) implementation("com.google.mlkit:genai-prompt:1.0.0-beta2") @@ -208,6 +223,13 @@ kotlin { // WorkManager — periodic background git sync implementation("androidx.work:work-runtime-ktx:2.9.1") + // Kable — Kotlin coroutine BLE for Android (GATT scanning and peripheral management) + implementation("com.juul.kable:core:0.32.0") + + // ONNX Runtime — monocular depth inference (Depth Anything V2 ViT-S) + // Model (~100MB) is downloaded on first use via DepthModelDownloader, not bundled. + implementation("com.microsoft.onnxruntime:onnxruntime-android:1.20.0") + // JGit 5.13.x — Android git operations (Android-safe; Java 11 APIs with desugaring) implementation("org.eclipse.jgit:org.eclipse.jgit:5.13.3.202401111512-r") // JGit SSH/JSch integration module (provides JschConfigSessionFactory) @@ -262,6 +284,9 @@ kotlin { // Ktor engine for iOS (used by coil-network-ktor3) implementation("io.ktor:ktor-client-darwin:3.1.3") + + // Kable — Kotlin coroutine BLE for iOS/Apple targets (CoreBluetooth wrapper) + implementation("com.juul.kable:core:0.32.0") } } diff --git a/kmp/src/androidMain/AndroidManifest.xml b/kmp/src/androidMain/AndroidManifest.xml index 8aea06f0..cf10c4ff 100644 --- a/kmp/src/androidMain/AndroidManifest.xml +++ b/kmp/src/androidMain/AndroidManifest.xml @@ -1,6 +1,54 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/google/AndroidGoogleAuthManager.kt b/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/google/AndroidGoogleAuthManager.kt new file mode 100644 index 00000000..4afbfcd9 --- /dev/null +++ b/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/google/AndroidGoogleAuthManager.kt @@ -0,0 +1,85 @@ +// Copyright (c) 2026 Tyler Stapler +// SPDX-License-Identifier: Elastic-2.0 + +package dev.stapler.stelekit.platform.google + +import android.content.Intent +import android.net.Uri +import arrow.core.Either +import arrow.core.left +import arrow.core.right +import dev.stapler.stelekit.error.DomainError +import dev.stapler.stelekit.platform.SteleKitContext + +/** + * Android implementation of [GoogleAuthManager]. + * + * Uses a WebView/Custom Tab OAuth 2.0 flow targeting: + * https://accounts.google.com/o/oauth2/auth + * + * Credential Manager (play-services-auth) is NOT yet available in the project dependencies. + * This stub opens the OAuth consent screen in the system browser (Custom Tab or fallback). + * The redirect URI must be registered in the Google Cloud Console as: + * com.stelekit.app:/oauth2redirect + * + * TODO (Story 7.2): When `credentials` + `credentials-play-services-auth` deps are added + * to build.gradle.kts, replace this with GetGoogleIdOption Credential Manager flow. + * + * NOTE: Token exchange (auth code → access/refresh tokens) requires a server-side endpoint + * or a native app client secret. For now this implementation is a structural stub that + * opens the browser. The token exchange is wired once a client_id is configured. + */ +class AndroidGoogleAuthManager( + private val tokenStore: GoogleTokenStore, + private val clientId: String = "", +) : GoogleAuthManager { + + companion object { + val SCOPES = listOf( + "https://www.googleapis.com/auth/drive.file", + "email", + "profile", + ) + const val REDIRECT_URI = "com.stelekit.app:/oauth2redirect" + } + + override suspend fun authenticate(): Either { + if (clientId.isBlank()) { + return DomainError.NetworkError.HttpError( + statusCode = 400, + message = "Google OAuth client ID not configured. Set WEB_CLIENT_ID in build config.", + ).left() + } + + val scopeString = SCOPES.joinToString(" ") + val authUrl = "https://accounts.google.com/o/oauth2/auth" + + "?client_id=$clientId" + + "&redirect_uri=${Uri.encode(REDIRECT_URI)}" + + "&response_type=code" + + "&scope=${Uri.encode(scopeString)}" + + "&access_type=offline" + + "&prompt=consent" + + return try { + val intent = Intent(Intent.ACTION_VIEW, Uri.parse(authUrl)) + .addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + SteleKitContext.context.startActivity(intent) + // Browser launched — actual token storage happens via deep-link callback. + // Return a pending state; real token saving occurs in the deep-link handler. + DomainError.NetworkError.HttpError( + statusCode = 202, + message = "OAuth flow initiated in browser. Complete sign-in to continue.", + ).left() + } catch (e: Exception) { + if (e is kotlinx.coroutines.CancellationException) throw e + DomainError.NetworkError.HttpError( + statusCode = -1, + message = "Failed to launch OAuth browser: ${e.message}", + ).left() + } + } + + override suspend fun signOut() { + tokenStore.clearTokens() + } +} diff --git a/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/google/AndroidGoogleTokenStore.kt b/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/google/AndroidGoogleTokenStore.kt new file mode 100644 index 00000000..81911823 --- /dev/null +++ b/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/google/AndroidGoogleTokenStore.kt @@ -0,0 +1,92 @@ +// Copyright (c) 2026 Tyler Stapler +// SPDX-License-Identifier: Elastic-2.0 + +package dev.stapler.stelekit.platform.google + +import android.util.Log +import androidx.security.crypto.EncryptedSharedPreferences +import androidx.security.crypto.MasterKey +import dev.stapler.stelekit.platform.SteleKitContext +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext + +/** + * Android implementation of [GoogleTokenStore] using [EncryptedSharedPreferences] + * backed by Android Keystore (AES256-GCM keys). + * + * SECURITY: If [EncryptedSharedPreferences] fails to initialize (corrupted Keystore, + * missing hardware), this implementation throws rather than falling back to plain + * SharedPreferences — per the ADR-003 security requirement. + */ +class AndroidGoogleTokenStore : GoogleTokenStore { + + private val prefs by lazy { + try { + val masterKey = MasterKey.Builder(SteleKitContext.context) + .setKeyScheme(MasterKey.KeyScheme.AES256_GCM) + .build() + EncryptedSharedPreferences.create( + SteleKitContext.context, + "stelekit_google_tokens", + masterKey, + EncryptedSharedPreferences.PrefKeyEncryptionScheme.AES256_SIV, + EncryptedSharedPreferences.PrefValueEncryptionScheme.AES256_GCM, + ) + } catch (e: CancellationException) { + throw e + } catch (e: Exception) { + // Do NOT fall back to plain SharedPreferences for tokens — fail loudly. + Log.e(TAG, "EncryptedSharedPreferences initialization failed. Google tokens cannot be stored securely.", e) + throw IllegalStateException( + "Android Keystore unavailable. Cannot store OAuth tokens securely. " + + "Google account features require a device with hardware-backed Keystore.", + e, + ) + } + } + + override suspend fun saveTokens( + accessToken: String, + refreshToken: String, + expiresAt: Long, + ) = withContext(Dispatchers.IO) { + prefs.edit() + .putString(KEY_ACCESS_TOKEN, accessToken) + .putString(KEY_REFRESH_TOKEN, refreshToken) + .putLong(KEY_EXPIRES_AT, expiresAt) + .apply() + } + + override suspend fun getAccessToken(): String? = withContext(Dispatchers.IO) { + prefs.getString(KEY_ACCESS_TOKEN, null) + } + + override suspend fun getRefreshToken(): String? = withContext(Dispatchers.IO) { + prefs.getString(KEY_REFRESH_TOKEN, null) + } + + override suspend fun getExpiresAt(): Long? = withContext(Dispatchers.IO) { + if (!prefs.contains(KEY_EXPIRES_AT)) null + else prefs.getLong(KEY_EXPIRES_AT, 0L) + } + + override suspend fun clearTokens() = withContext(Dispatchers.IO) { + prefs.edit() + .remove(KEY_ACCESS_TOKEN) + .remove(KEY_REFRESH_TOKEN) + .remove(KEY_EXPIRES_AT) + .apply() + } + + override suspend fun isAuthenticated(): Boolean = withContext(Dispatchers.IO) { + prefs.contains(KEY_ACCESS_TOKEN) && prefs.getString(KEY_ACCESS_TOKEN, null) != null + } + + private companion object { + private const val TAG = "AndroidGoogleTokenStore" + private const val KEY_ACCESS_TOKEN = "google_access_token" + private const val KEY_REFRESH_TOKEN = "google_refresh_token" + private const val KEY_EXPIRES_AT = "google_expires_at" + } +} diff --git a/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/google/GooglePhotosPickerLauncher.kt b/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/google/GooglePhotosPickerLauncher.kt new file mode 100644 index 00000000..1b407eef --- /dev/null +++ b/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/google/GooglePhotosPickerLauncher.kt @@ -0,0 +1,155 @@ +// Copyright (c) 2026 Tyler Stapler +// SPDX-License-Identifier: Elastic-2.0 + +package dev.stapler.stelekit.platform.google + +import android.content.Intent +import android.net.Uri +import arrow.core.Either +import arrow.core.left +import arrow.core.right +import dev.stapler.stelekit.error.DomainError +import dev.stapler.stelekit.platform.SteleKitContext +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.delay + +/** + * Android implementation of the Google Photos Picker flow using the NEW Picker API + * (photospicker.googleapis.com), introduced as the only supported path after the + * photoslibrary.readonly scope was revoked in March 2025. + * + * Flow: + * 1. Call [PhotosPickerApiClient.createPhotosPickerSession] to get a picker session + URI. + * 2. Open the picker URI in a Custom Tab (system browser overlay — no WebView needed). + * 3. Poll [PhotosPickerApiClient.getPickerSession] until [PhotosPickerSession.mediaItemsSet] = true. + * 4. Download selected media bytes via [PhotosPickerApiClient.downloadPickerMedia] using the + * temporary `baseUrl` (NOT stored long-term — store `mediaItemId` instead). + * 5. Clean up the session via [PhotosPickerApiClient.deletePickerSession]. + * + * UI copy requirement (Story 7.5): callers must display + * "Select from Google Photos — you choose which specific photos to share with SteleKit" + * to clarify the limited-access scope to users (per post-March-2025 policy requirements). + * + * Prerequisites: user must be authenticated (call [GoogleAuthManager.authenticate] first). + * If not authenticated, [launchPicker] returns [DomainError.SensorError.PermissionDenied]. + */ +class GooglePhotosPickerLauncher( + private val apiClient: PhotosPickerApiClient, + private val tokenStore: GoogleTokenStore, +) { + + companion object { + /** + * UI copy to display to users before launching the picker. + * Required per post-March-2025 Google Photos scope restrictions. + */ + const val PICKER_UI_COPY = + "Select from Google Photos — you choose which specific photos to share with SteleKit" + + /** Maximum number of polling attempts before giving up (60 × 2s = 120s timeout). */ + private const val MAX_POLL_ATTEMPTS = 60 + + /** Polling interval in milliseconds. */ + private const val POLL_INTERVAL_MS = 2_000L + } + + /** + * Launch the Google Photos Picker and return the selected photo bytes. + * + * This suspend function: + * 1. Creates a picker session. + * 2. Opens the picker URI in a system browser / Custom Tab. + * 3. Polls for user selection (up to [MAX_POLL_ATTEMPTS] × [POLL_INTERVAL_MS] = 120s). + * 4. Downloads the selected photo bytes. + * 5. Returns the bytes and the stable [mediaItemId] for long-term storage. + * + * @return Pair of (imageBytes, mediaItemId) on success. + */ + suspend fun launchPicker(): Either> { + if (!tokenStore.isAuthenticated()) { + return DomainError.SensorError.PermissionDenied( + "Google account not connected. Connect a Google account first to import from Google Photos.", + ).left() + } + + // Step 1: Create picker session + val session = apiClient.createPhotosPickerSession().getOrNull() + ?: return DomainError.NetworkError.HttpError( + statusCode = -1, + message = "Failed to create Google Photos Picker session. Check network connection.", + ).left() + + // Step 2: Open picker URI in system browser + try { + val pickerIntent = Intent(Intent.ACTION_VIEW, Uri.parse(session.pickerUri)) + .addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + SteleKitContext.context.startActivity(pickerIntent) + } catch (e: Exception) { + if (e is CancellationException) throw e + return DomainError.NetworkError.HttpError( + statusCode = -1, + message = "Failed to open Google Photos Picker: ${e.message}", + ).left() + } + + // Step 3: Poll until user makes a selection + var attempts = 0 + var latestSession = session + while (!latestSession.mediaItemsSet && attempts < MAX_POLL_ATTEMPTS) { + delay(POLL_INTERVAL_MS) + attempts++ + val polled = apiClient.getPickerSession(session.id).getOrNull() ?: break + latestSession = polled + } + + if (!latestSession.mediaItemsSet) { + apiClient.deletePickerSession(session.id) + return DomainError.NetworkError.HttpError( + statusCode = 408, + message = "Google Photos Picker timed out or was cancelled.", + ).left() + } + + // Step 4: Get media items from the completed session + // The session response should contain mediaItems — re-fetch the session with mediaItems field + val mediaItems = fetchSessionMediaItems(session.id) + val firstItem = mediaItems.firstOrNull() ?: run { + apiClient.deletePickerSession(session.id) + return DomainError.NetworkError.HttpError( + statusCode = -1, + message = "No photo selected from Google Photos.", + ).left() + } + + // Step 5: Download the selected photo bytes using the temporary baseUrl + val bytes = apiClient.downloadPickerMedia( + baseUrl = firstItem.first, // temporary baseUrl — NOT stored + mediaItemId = firstItem.second, + ).getOrNull() ?: run { + apiClient.deletePickerSession(session.id) + return DomainError.NetworkError.HttpError( + statusCode = -1, + message = "Failed to download selected photo from Google Photos.", + ).left() + } + + // Step 6: Clean up the session + apiClient.deletePickerSession(session.id) + + // Return bytes + stable mediaItemId (store this, NOT the baseUrl) + return Pair(bytes, firstItem.second).right() + } + + /** + * Fetch the media items (baseUrl, mediaItemId pairs) from a completed picker session. + * + * Delegates to [PhotosPickerApiClient.listPickerMediaItems] which calls: + * GET https://photospicker.googleapis.com/v1/mediaItems?sessionId={id} + * Response: { "mediaItems": [{ "id": "...", "mediaFile": { "baseUrl": "...", ... } }] } + * + * Returns an empty list if the request fails. + */ + private suspend fun fetchSessionMediaItems(sessionId: String): List> { + return apiClient.listPickerMediaItems(sessionId).getOrNull() ?: emptyList() + } +} diff --git a/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/measurement/ble/AndroidMeasurementForegroundService.kt b/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/measurement/ble/AndroidMeasurementForegroundService.kt new file mode 100644 index 00000000..84838f43 --- /dev/null +++ b/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/measurement/ble/AndroidMeasurementForegroundService.kt @@ -0,0 +1,47 @@ +package dev.stapler.stelekit.platform.measurement.ble + +import android.app.Service +import android.content.Intent +import android.os.IBinder + +/** + * Foreground service stub for managing BLE measurement connections on Android API 31+. + * + * Android requires a foreground service with type `connectedDevice` for BLE GATT connections + * initiated while the app is in the background (API 31+). This stub satisfies the manifest + * registration requirement and the compilation dependency without containing BLE logic. + * + * To activate: + * 1. Add Kable dependency to build.gradle.kts. + * 2. Inject [KableBleScanner] and the active [ExternalMeasurementDevice] into this service. + * 3. Show a persistent notification with "Measuring…" text when a device is CONNECTED. + * 4. Call [stopForeground] and [stopSelf] when the last device disconnects. + * + * Notification requirements (Android 13+): + * - POST_NOTIFICATIONS permission must be granted before starting this service. + * - Notification must use a dedicated measurement notification channel. + * - Notification is auto-dismissed on [DeviceConnectionState.DISCONNECTED]. + * + * The service is declared in AndroidManifest.xml with: + * `android:foregroundServiceType="connectedDevice"` + */ +class AndroidMeasurementForegroundService : Service() { + + override fun onBind(intent: Intent?): IBinder? = null + + override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { + // TODO(BLE): Start foreground notification and connect active BLE device. + // val notification = buildMeasuringNotification() + // startForeground(NOTIFICATION_ID, notification, FOREGROUND_SERVICE_TYPE_CONNECTED_DEVICE) + return START_NOT_STICKY + } + + override fun onDestroy() { + super.onDestroy() + // TODO(BLE): Disconnect active BLE device and release Kable scanner. + } + + companion object { + const val NOTIFICATION_ID: Int = 9001 + } +} diff --git a/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/measurement/ble/BoschGlmKableDevice.kt b/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/measurement/ble/BoschGlmKableDevice.kt new file mode 100644 index 00000000..4a4f3f92 --- /dev/null +++ b/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/measurement/ble/BoschGlmKableDevice.kt @@ -0,0 +1,140 @@ +package dev.stapler.stelekit.platform.measurement.ble + +import android.content.Context +import android.content.Intent +import arrow.core.Either +import arrow.core.left +import arrow.core.right +import com.juul.kable.Advertisement +import com.juul.kable.characteristicOf +import com.juul.kable.peripheral +import dev.stapler.stelekit.error.DomainError +import dev.stapler.stelekit.platform.measurement.DeviceConnectionState +import dev.stapler.stelekit.platform.measurement.ExternalMeasurementDevice +import dev.stapler.stelekit.platform.measurement.MeasurementReading +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.MutableSharedFlow +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asSharedFlow +import kotlinx.coroutines.launch +import java.io.IOException +import kotlin.math.min + +/** + * [ExternalMeasurementDevice] implementation for Bosch GLM laser rangefinders. + * + * The Bosch GLM uses SPP-over-BLE (Serial Port Profile emulation), sending ASCII + * measurement strings in the format `MM:D\r\n`. + * + * GATT error 133 retry logic mirrors [LeicaDistoKableDevice]: exponential backoff + * starting at 2 s, capped at 60 s, max 5 attempts. + */ +class BoschGlmKableDevice( + private val advertisement: Advertisement, + private val context: Context, +) : ExternalMeasurementDevice { + + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) + private val peripheral = scope.peripheral(advertisement) + + private val _connectionState = MutableStateFlow(DeviceConnectionState.DISCONNECTED) + override val connectionState: StateFlow = _connectionState + + private val _measurements = MutableSharedFlow(extraBufferCapacity = 16) + + override val deviceName: String + get() = advertisement.name ?: "Bosch GLM" + + override val deviceId: String + get() = advertisement.identifier + + override fun measurementFlow(): Flow = _measurements.asSharedFlow() + + override suspend fun connect(): Either { + context.startForegroundService( + Intent(context, AndroidMeasurementForegroundService::class.java), + ) + _connectionState.value = DeviceConnectionState.CONNECTING + var attempt = 0 + while (attempt < MAX_RETRIES) { + try { + peripheral.connect() + _connectionState.value = DeviceConnectionState.CONNECTED + launchMeasurementLoop() + return Unit.right() + } catch (e: IOException) { + val msg = e.message ?: "" + if (msg.contains("133") || msg.contains("GATT")) { + attempt++ + if (attempt >= MAX_RETRIES) { + _connectionState.value = DeviceConnectionState.ERROR + return DomainError.BleError.Gatt133( + attempts = MAX_RETRIES, + message = e.message ?: "GATT error", + ).left() + } + val backoffMs = min(BACKOFF_BASE_MS shl (attempt - 1), BACKOFF_MAX_MS) + delay(backoffMs.toLong()) + try { peripheral.disconnect() } catch (de: Exception) { if (de is CancellationException) throw de } + } else { + _connectionState.value = DeviceConnectionState.ERROR + return DomainError.BleError.ConnectionFailed(e.message ?: "connect failed").left() + } + } catch (e: Exception) { + if (e is CancellationException) throw e + _connectionState.value = DeviceConnectionState.ERROR + return DomainError.BleError.ConnectionFailed(e.message ?: "connect failed").left() + } + } + _connectionState.value = DeviceConnectionState.ERROR + return DomainError.BleError.ConnectionFailed("Max retries exceeded").left() + } + + override suspend fun disconnect() { + try { + peripheral.disconnect() + } finally { + _connectionState.value = DeviceConnectionState.DISCONNECTED + } + } + + private fun launchMeasurementLoop() { + scope.launch { + try { + val sppCharacteristic = characteristicOf( + service = BoschGlmProtocol.SPP_SERVICE_UUID, + characteristic = SPP_CHARACTERISTIC_UUID, + ) + peripheral.observe(sppCharacteristic).collect { bytes: ByteArray -> + val text = String(bytes, Charsets.UTF_8) + val reading = BoschGlmProtocol.parseResponse(text, deviceId) + if (reading != null) { + _measurements.tryEmit(reading) + } + } + } catch (le: Exception) { + if (le is CancellationException) throw le + _connectionState.value = DeviceConnectionState.ERROR + } + } + } + + companion object { + private const val MAX_RETRIES = 5 + private const val BACKOFF_BASE_MS = 2_000 + private const val BACKOFF_MAX_MS = 60_000 + + /** + * SPP data transfer characteristic UUID. + * Bosch GLM uses the standard SPP service (0x1101) with a vendor + * characteristic for data transfer. + */ + private const val SPP_CHARACTERISTIC_UUID = "00001101-0000-1000-8000-00805f9b34fb" + } +} diff --git a/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/measurement/ble/KableBleScanner.kt b/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/measurement/ble/KableBleScanner.kt new file mode 100644 index 00000000..4bfa7def --- /dev/null +++ b/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/measurement/ble/KableBleScanner.kt @@ -0,0 +1,74 @@ +package dev.stapler.stelekit.platform.measurement.ble + +import android.Manifest +import android.content.Context +import android.content.pm.PackageManager +import android.os.Build +import android.util.Log +import androidx.core.content.ContextCompat +import com.juul.kable.Scanner +import dev.stapler.stelekit.platform.measurement.ExternalMeasurementDevice +import dev.stapler.stelekit.platform.measurement.MeasurementDeviceFactory +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.emptyFlow +import kotlinx.coroutines.flow.mapNotNull + +/** + * Android BLE scanner backed by Kable (com.juul.kable). + * + * Scans for: + * - Leica DISTO devices identified by advertised service UUID prefix + * - Bosch GLM devices identified by device name prefix ("Bosch" or "GLM") + * + * Required Android manifest permissions (declared in AndroidManifest.xml): + * - android.permission.BLUETOOTH_SCAN (API 31+) + * - android.permission.BLUETOOTH_CONNECT (API 31+) + * - android.hardware.bluetooth_le (uses-feature) + * + * GATT pitfalls handled by device implementations: + * - GATT 133 exponential backoff: 2 s base, 60 s max, 5 retries max + * - peripheral.disconnect() on every exit path (prevents 30-object GATT leak) + * - MTU negotiated to 100 bytes before first characteristic read (Leica only) + */ +class KableBleScanner( + private val context: Context, +) : MeasurementDeviceFactory { + + override fun scan(): Flow { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { + val scanGranted = ContextCompat.checkSelfPermission( + context, + Manifest.permission.BLUETOOTH_SCAN, + ) == PackageManager.PERMISSION_GRANTED + val connectGranted = ContextCompat.checkSelfPermission( + context, + Manifest.permission.BLUETOOTH_CONNECT, + ) == PackageManager.PERMISSION_GRANTED + if (!scanGranted || !connectGranted) { + Log.w( + TAG, + "BLE permissions not granted — scan() returning empty flow. " + + "Request BLUETOOTH_SCAN and BLUETOOTH_CONNECT before calling scan().", + ) + return emptyFlow() + } + } + + return Scanner().advertisements + .mapNotNull { advertisement -> + val name = advertisement.name ?: "" + val serviceUuids = advertisement.uuids.map { it.toString().lowercase() } + when { + serviceUuids.any { it.startsWith(LeicaDistoProtocol.SERVICE_UUID_PREFIX.lowercase()) } -> + LeicaDistoKableDevice(advertisement, context) + name.startsWith("GLM") || name.startsWith("Bosch") -> + BoschGlmKableDevice(advertisement, context) + else -> null + } + } + } + + companion object { + private const val TAG = "KableBleScanner" + } +} diff --git a/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/measurement/ble/LeicaDistoKableDevice.kt b/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/measurement/ble/LeicaDistoKableDevice.kt new file mode 100644 index 00000000..56cf875a --- /dev/null +++ b/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/measurement/ble/LeicaDistoKableDevice.kt @@ -0,0 +1,147 @@ +package dev.stapler.stelekit.platform.measurement.ble + +import android.content.Context +import android.content.Intent +import arrow.core.Either +import arrow.core.left +import arrow.core.right +import com.juul.kable.Advertisement +import com.juul.kable.AndroidPeripheral +import com.juul.kable.WriteType +import com.juul.kable.characteristicOf +import com.juul.kable.peripheral +import dev.stapler.stelekit.error.DomainError +import dev.stapler.stelekit.platform.measurement.DeviceConnectionState +import dev.stapler.stelekit.platform.measurement.ExternalMeasurementDevice +import dev.stapler.stelekit.platform.measurement.MeasurementReading +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.MutableSharedFlow +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asSharedFlow +import kotlinx.coroutines.launch +import java.io.IOException +import kotlin.math.min + +/** + * [ExternalMeasurementDevice] implementation for Leica DISTO laser rangefinders. + * + * Uses Kable for GATT connection management. Subscribes to the measurement notification + * characteristic and writes an ACK within 2 seconds per protocol requirement. + * + * GATT error 133 retry logic: exponential backoff starting at 2 s, capped at 60 s, + * max 5 attempts before emitting [DeviceConnectionState.ERROR]. + */ +class LeicaDistoKableDevice( + private val advertisement: Advertisement, + private val context: Context, +) : ExternalMeasurementDevice { + + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) + private val peripheral = scope.peripheral(advertisement) + + private val _connectionState = MutableStateFlow(DeviceConnectionState.DISCONNECTED) + override val connectionState: StateFlow = _connectionState + + private val _measurements = MutableSharedFlow(extraBufferCapacity = 16) + + override val deviceName: String + get() = advertisement.name ?: "Leica DISTO" + + override val deviceId: String + get() = advertisement.identifier + + override fun measurementFlow(): Flow = _measurements.asSharedFlow() + + override suspend fun connect(): Either { + context.startForegroundService( + Intent(context, AndroidMeasurementForegroundService::class.java), + ) + _connectionState.value = DeviceConnectionState.CONNECTING + var attempt = 0 + while (attempt < MAX_RETRIES) { + try { + peripheral.connect() + // requestMtu is AndroidPeripheral-specific + (peripheral as? AndroidPeripheral)?.requestMtu(100) + _connectionState.value = DeviceConnectionState.CONNECTED + launchMeasurementLoop() + return Unit.right() + } catch (e: IOException) { + val msg = e.message ?: "" + if (msg.contains("133") || msg.contains("GATT")) { + attempt++ + if (attempt >= MAX_RETRIES) { + _connectionState.value = DeviceConnectionState.ERROR + return DomainError.BleError.Gatt133( + attempts = MAX_RETRIES, + message = e.message ?: "GATT error", + ).left() + } + val backoffMs = min(BACKOFF_BASE_MS shl (attempt - 1), BACKOFF_MAX_MS) + delay(backoffMs.toLong()) + try { peripheral.disconnect() } catch (de: Exception) { if (de is CancellationException) throw de } + } else { + _connectionState.value = DeviceConnectionState.ERROR + return DomainError.BleError.ConnectionFailed(e.message ?: "connect failed").left() + } + } catch (e: Exception) { + if (e is CancellationException) throw e + _connectionState.value = DeviceConnectionState.ERROR + return DomainError.BleError.ConnectionFailed(e.message ?: "connect failed").left() + } + } + _connectionState.value = DeviceConnectionState.ERROR + return DomainError.BleError.ConnectionFailed("Max retries exceeded").left() + } + + override suspend fun disconnect() { + try { + peripheral.disconnect() + } finally { + _connectionState.value = DeviceConnectionState.DISCONNECTED + } + } + + private fun launchMeasurementLoop() { + scope.launch { + try { + val measureCharacteristic = characteristicOf( + service = LeicaDistoProtocol.SERVICE_UUID, + characteristic = LeicaDistoProtocol.MEASUREMENT_CHARACTERISTIC_UUID, + ) + val ackCharacteristic = characteristicOf( + service = LeicaDistoProtocol.SERVICE_UUID, + characteristic = LeicaDistoProtocol.ACK_CHARACTERISTIC_UUID, + ) + peripheral.observe(measureCharacteristic).collect { bytes: ByteArray -> + val reading = LeicaDistoProtocol.parseNotification(bytes, deviceId) + if (reading != null) { + _measurements.tryEmit(reading) + } + // Must ACK within 2 seconds regardless of parse result. + try { + peripheral.write(ackCharacteristic, LeicaDistoProtocol.ACK_BYTES, WriteType.WithResponse) + } catch (ae: Exception) { + if (ae is CancellationException) throw ae + /* best effort — don't crash the observe loop */ + } + } + } catch (le: Exception) { + if (le is CancellationException) throw le + _connectionState.value = DeviceConnectionState.ERROR + } + } + } + + companion object { + private const val MAX_RETRIES = 5 + private const val BACKOFF_BASE_MS = 2_000 + private const val BACKOFF_MAX_MS = 60_000 + } +} diff --git a/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/measurement/usb/AndroidUsbSerialFactory.kt b/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/measurement/usb/AndroidUsbSerialFactory.kt new file mode 100644 index 00000000..11b7c1be --- /dev/null +++ b/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/measurement/usb/AndroidUsbSerialFactory.kt @@ -0,0 +1,60 @@ +package dev.stapler.stelekit.platform.measurement.usb + +import android.app.PendingIntent +import android.content.BroadcastReceiver +import android.content.Context +import android.content.Intent +import android.hardware.usb.UsbManager +import dev.stapler.stelekit.platform.measurement.ExternalMeasurementDevice +import dev.stapler.stelekit.platform.measurement.MeasurementDeviceFactory +import dev.stapler.stelekit.platform.measurement.ble.NoOpBleScanner +import kotlinx.coroutines.flow.Flow + +/** + * Android USB serial (OTG) factory stub. + * + * NOTE: `usb-serial-for-android` (kai-morich fork) is NOT currently on the classpath. + * This stub delegates to [NoOpBleScanner] and returns an empty Flow. + * + * To enable USB serial support: + * 1. Add to androidMain in kmp/build.gradle.kts: + * `implementation("com.github.kai-morich:usb-serial-for-android:")` + * 2. Confirm LGPL 2.1 compliance via dynamic `.aar` linking (confirmed acceptable per plan ADR-004). + * 3. Replace the [scan] body with: + * ```kotlin + * val usbManager = context.getSystemService(Context.USB_SERVICE) as UsbManager + * val drivers = UsbSerialProber.getDefaultProber().findAllDrivers(usbManager) + * return flow { + * drivers.forEach { driver -> + * requestPermission(usbManager, driver.device) + * emit(AndroidUsbSerialDevice(driver, usbManager, context)) + * } + * } + * ``` + * 4. Implement `AndroidUsbSerialDevice` wrapping the driver's serial port in a Flow. + * + * USB_PERMISSION broadcast action for permission request/response: + * [USB_PERMISSION_ACTION] + * + * To request permission before opening a device, broadcast a [PendingIntent] via: + * `usbManager.requestPermission(usbDevice, pendingIntent)` + * and receive the response in a [BroadcastReceiver] registered for [USB_PERMISSION_ACTION]. + */ +class AndroidUsbSerialFactory( + @Suppress("UnusedPrivateMember") + private val context: Context, +) : MeasurementDeviceFactory { + + /** + * Custom action string for the USB_PERMISSION PendingIntent broadcast. + */ + companion object { + const val USB_PERMISSION_ACTION: String = + "dev.stapler.stelekit.USB_PERMISSION" + } + + // TODO(USB): Replace with real implementation once usb-serial-for-android is added. + private val noOp = NoOpBleScanner() + + override fun scan(): Flow = noOp.scan() +} diff --git a/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/ml/DepthModelDownloader.kt b/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/ml/DepthModelDownloader.kt new file mode 100644 index 00000000..ddc28641 --- /dev/null +++ b/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/ml/DepthModelDownloader.kt @@ -0,0 +1,164 @@ +package dev.stapler.stelekit.platform.ml + +import android.app.DownloadManager +import android.content.BroadcastReceiver +import android.content.Context +import android.content.Intent +import android.content.IntentFilter +import android.net.Uri +import arrow.core.Either +import arrow.core.left +import arrow.core.right +import dev.stapler.stelekit.error.DomainError +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.suspendCancellableCoroutine +import java.io.File +import kotlin.coroutines.resume + +/** + * Downloads the Depth Anything V2 ViT-S ONNX model on first use via Android's [DownloadManager]. + * + * Using [DownloadManager] (system service) rather than OkHttp/Ktor ensures the download survives + * process death — the system service continues the transfer even if the app is backgrounded. + * + * Model is stored at [Context.filesDir]/models/depth_anything_v2_small.onnx so the path is + * stable across restarts (unlike cacheDir, which can be cleared by the OS). + * + * @param context application context + */ +class DepthModelDownloader(private val context: Context) { + + private val modelFile: File = + File(context.filesDir, "models/depth_anything_v2_small.onnx") + + private val _modelState = MutableStateFlow(resolveInitialState()) + val modelState: StateFlow = _modelState.asStateFlow() + + /** Current in-flight [DownloadManager] enqueue ID, or -1 when idle. */ + private var activeDownloadId: Long = -1L + + // ── Public API ──────────────────────────────────────────────────────────── + + /** + * Ensure the model file is present. + * + * If the file already exists and passes the sanity-size check (> 10 MB), transitions to + * [ModelState.Ready] and returns it immediately. Otherwise starts the download and suspends + * until [DownloadManager] signals completion via [DownloadManager.ACTION_DOWNLOAD_COMPLETE]. + * + * Must be called from a coroutine (suspend function). Safe to call multiple times. + */ + suspend fun downloadModel(): Either { + // Fast path: model already present and non-corrupt. + if (isModelReady()) { + _modelState.value = ModelState.Ready + return modelFile.right() + } + + // Ensure destination directory exists. + modelFile.parentFile?.mkdirs() + + return suspendCancellableCoroutine { continuation -> + val downloadManager = + context.getSystemService(Context.DOWNLOAD_SERVICE) as DownloadManager + + val request = DownloadManager.Request(Uri.parse(MODEL_URL)).apply { + setTitle("Depth model") + setDescription("Downloading depth estimation model (~100 MB)") + setDestinationUri(Uri.fromFile(modelFile)) + setNotificationVisibility(DownloadManager.Request.VISIBILITY_VISIBLE) + setAllowedOverMetered(true) + setAllowedOverRoaming(false) + } + + val downloadId = downloadManager.enqueue(request) + activeDownloadId = downloadId + _modelState.value = ModelState.Downloading(progress = 0) + + // BroadcastReceiver — fires when this download (or any other) completes. + val receiver = object : BroadcastReceiver() { + override fun onReceive(ctx: Context?, intent: Intent?) { + val completedId = + intent?.getLongExtra(DownloadManager.EXTRA_DOWNLOAD_ID, -1L) ?: -1L + if (completedId != downloadId) return // not our download + + context.unregisterReceiver(this) + activeDownloadId = -1L + + val query = DownloadManager.Query().setFilterById(downloadId) + val cursor = downloadManager.query(query) + val succeeded = cursor?.use { c -> + if (c.moveToFirst()) { + val statusCol = c.getColumnIndex(DownloadManager.COLUMN_STATUS) + c.getInt(statusCol) == DownloadManager.STATUS_SUCCESSFUL + } else false + } ?: false + + if (succeeded && isModelReady()) { + _modelState.value = ModelState.Ready + continuation.resume(modelFile.right()) + } else { + _modelState.value = ModelState.Failed + continuation.resume( + DomainError.SensorError.HardwareUnavailable( + "Depth model download failed", + ).left(), + ) + } + } + } + + context.registerReceiver( + receiver, + IntentFilter(DownloadManager.ACTION_DOWNLOAD_COMPLETE), + Context.RECEIVER_NOT_EXPORTED, + ) + + // Cancel the download if the coroutine scope is cancelled. + continuation.invokeOnCancellation { + runCatching { context.unregisterReceiver(receiver) } + downloadManager.remove(downloadId) + _modelState.value = ModelState.Absent + activeDownloadId = -1L + } + } + } + + /** Absolute path of the model file. Safe to pass to [ai.onnxruntime.OrtSession]. */ + fun modelFilePath(): String = modelFile.absolutePath + + /** True when the model file exists on disk and exceeds the minimum sanity size. */ + fun isModelReady(): Boolean = modelFile.exists() && modelFile.length() > MIN_MODEL_SIZE_BYTES + + // ── Sealed state hierarchy ──────────────────────────────────────────────── + + /** Download lifecycle state exposed as [StateFlow]. */ + sealed interface ModelState { + /** No model file on disk. */ + data object Absent : ModelState + + /** Download in progress — [progress] is 0–100, or -1 if indeterminate. */ + data class Downloading(val progress: Int) : ModelState + + /** Model file present and verified. */ + data object Ready : ModelState + + /** Download failed or model file corrupt. */ + data object Failed : ModelState + } + + // ── Private helpers ─────────────────────────────────────────────────────── + + private fun resolveInitialState(): ModelState = + if (isModelReady()) ModelState.Ready else ModelState.Absent + + companion object { + const val MODEL_URL = + "https://huggingface.co/onnx-community/depth-anything-v2-small/resolve/main/onnx/model.onnx" + + /** Sanity threshold: a valid model must be larger than 10 MB. */ + private const val MIN_MODEL_SIZE_BYTES = 10L * 1024 * 1024 + } +} diff --git a/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/ml/OnnxMonocularDepthEstimator.kt b/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/ml/OnnxMonocularDepthEstimator.kt new file mode 100644 index 00000000..1a84b6e2 --- /dev/null +++ b/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/ml/OnnxMonocularDepthEstimator.kt @@ -0,0 +1,214 @@ +package dev.stapler.stelekit.platform.ml + +import ai.onnxruntime.OnnxTensor +import ai.onnxruntime.OrtEnvironment +import ai.onnxruntime.OrtSession +import android.app.ActivityManager +import android.content.Context +import android.graphics.Bitmap +import android.os.Build +import androidx.compose.ui.graphics.ImageBitmap +import androidx.compose.ui.graphics.asAndroidBitmap +import arrow.core.Either +import arrow.core.left +import arrow.core.right +import dev.stapler.stelekit.error.DomainError +import dev.stapler.stelekit.coroutines.PlatformDispatcher +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.withContext +import java.nio.FloatBuffer + +/** + * ONNX Runtime implementation of [MonocularDepthEstimator] for Android. + * + * Prerequisites: + * - API 26+ (Android 8.0 Oreo) + * - ≥ 3 GB total RAM (memory gate) + * - Model downloaded via [DepthModelDownloader] to `filesDir/models/depth_anything_v2_small.onnx` + * + * ADR-005: All results carry 15% confidence. The UI must show + * "Low confidence — verify with reference object" whenever this estimator is active. + * + * All ORT calls are wrapped in try/catch — ORT may throw [ai.onnxruntime.OrtException] or + * [UnsatisfiedLinkError] on unsupported devices. + */ +class OnnxMonocularDepthEstimator(private val context: Context) : MonocularDepthEstimator { + + private var ortEnv: OrtEnvironment? = null + private var ortSession: OrtSession? = null + private var _isAvailable: Boolean = false + + private val inputBuffer: FloatBuffer by lazy { + FloatBuffer.allocate(INPUT_SIZE * INPUT_SIZE * 3) + } + + val downloader: DepthModelDownloader = DepthModelDownloader(context) + + /** Mirrors [DepthModelDownloader.modelState] for UI observation. */ + val modelState: StateFlow = downloader.modelState + + override val isAvailable: Boolean + get() = _isAvailable + + /** + * Initialize the ONNX session. + * + * Returns [DomainError.SensorError.HardwareUnavailable] if: + * - device is below API 26 + * - device has < 3 GB RAM + * - model file is not yet present (caller should offer download prompt) + * + * On NNAPI failure, falls back silently to CPU inference. + */ + override suspend fun initialize(): Either { + // Already initialized — idempotent. + if (_isAvailable && ortSession != null) return Unit.right() + + // API level gate. + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) { + return DomainError.SensorError.HardwareUnavailable( + "OnnxMonocularDepthEstimator requires API 26+", + ).left() + } + + // Memory gate: require ≥ 3 GB total RAM. + val am = context.getSystemService(Context.ACTIVITY_SERVICE) as ActivityManager + val memInfo = ActivityManager.MemoryInfo() + am.getMemoryInfo(memInfo) + if (memInfo.totalMem < MIN_RAM_BYTES) { + return DomainError.SensorError.HardwareUnavailable( + "OnnxMonocularDepthEstimator requires ≥ 3 GB RAM " + + "(device has ${memInfo.totalMem / (1024 * 1024)} MB)", + ).left() + } + + // Model availability gate — caller shows "Download model" prompt on HardwareUnavailable. + if (!downloader.isModelReady()) { + return DomainError.SensorError.HardwareUnavailable( + "Depth model not downloaded", + ).left() + } + + return withContext(PlatformDispatcher.Default) { + try { + val env = OrtEnvironment.getEnvironment() + ortEnv = env + + // Try NNAPI hardware delegate first; fall back to CPU if unavailable. + val session = try { + val opts = OrtSession.SessionOptions().apply { addNnapi() } + env.createSession(downloader.modelFilePath(), opts) + } catch (ne: Exception) { + if (ne is CancellationException) throw ne + // NNAPI not available on this device — use CPU inference. + env.createSession(downloader.modelFilePath(), OrtSession.SessionOptions()) + } + + ortSession = session + _isAvailable = true + Unit.right() + } catch (e: Exception) { + if (e is CancellationException) throw e + _isAvailable = false + DomainError.SensorError.HardwareUnavailable( + "Failed to create ONNX session: ${e.message}", + ).left() + } + } + } + + /** + * Run Depth Anything V2 ViT-S inference on [imageBitmap]. + * + * Steps: + * 1. Resize to 518×518 (model's fixed input resolution) + * 2. Normalize with ImageNet mean/std + * 3. Run inference via ORT + * 4. Normalize output to [0,1] range (divide by max) + * + * Returns a flat [FloatArray] of 518×518 relative depth values in row-major order. + * Values are relative, not metric — must be scaled by a known reference distance. + * + * NEVER called on the main thread — dispatched to [PlatformDispatcher.Default]. + */ + override suspend fun estimateDepth(imageBitmap: ImageBitmap): Either { + val session = ortSession + ?: return DomainError.SensorError.HardwareUnavailable( + "OnnxMonocularDepthEstimator not initialized", + ).left() + + return withContext(PlatformDispatcher.Default) { + try { + val env = ortEnv ?: OrtEnvironment.getEnvironment() + val androidBitmap: Bitmap = imageBitmap.asAndroidBitmap() + + // 1. Resize to model input size (518×518). + val resized = Bitmap.createScaledBitmap(androidBitmap, INPUT_SIZE, INPUT_SIZE, true) + + // 2. Build CHW float buffer with ImageNet normalization. + val pixels = IntArray(INPUT_SIZE * INPUT_SIZE) + resized.getPixels(pixels, 0, INPUT_SIZE, 0, 0, INPUT_SIZE, INPUT_SIZE) + resized.recycle() + + // Layout: [1, 3, 518, 518] in NCHW order — fill R plane, then G, then B. + inputBuffer.rewind() + for (pixelValue in pixels) { + val r = ((pixelValue shr 16) and 0xFF) / 255f + inputBuffer.put((r - MEAN_R) / STD_R) + } + for (pixelValue in pixels) { + val g = ((pixelValue shr 8) and 0xFF) / 255f + inputBuffer.put((g - MEAN_G) / STD_G) + } + for (pixelValue in pixels) { + val b = (pixelValue and 0xFF) / 255f + inputBuffer.put((b - MEAN_B) / STD_B) + } + inputBuffer.rewind() + + // 3. Create input tensor and run inference. + val inputShape = longArrayOf(1L, 3L, INPUT_SIZE.toLong(), INPUT_SIZE.toLong()) + val inputTensor = OnnxTensor.createTensor(env, inputBuffer, inputShape) + + val outputs = inputTensor.use { + session.run(mapOf("image" to inputTensor)) + } + + // 4. Extract depth output — shape [1, 1, 518, 518]. + val rawDepth: FloatArray = outputs.use { result -> + @Suppress("UNCHECKED_CAST") + (result["depth"].get().value as Array>)[0][0] + } + + // 5. Normalize to [0,1]. + val maxVal = rawDepth.max() + if (maxVal > 0f) { + for (i in rawDepth.indices) rawDepth[i] /= maxVal + } + + rawDepth.right() + } catch (e: Exception) { + if (e is CancellationException) throw e + DomainError.SensorError.CaptureFailed( + "ONNX depth inference failed: ${e.message}", + ).left() + } + } + } + + companion object { + private const val INPUT_SIZE = 518 + + // ImageNet normalization constants. + private const val MEAN_R = 0.485f + private const val MEAN_G = 0.456f + private const val MEAN_B = 0.406f + private const val STD_R = 0.229f + private const val STD_G = 0.224f + private const val STD_B = 0.225f + + /** Minimum required RAM: 3 GB. */ + private const val MIN_RAM_BYTES = 3L * 1024L * 1024L * 1024L + } +} diff --git a/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/sensor/ARCoreDepthProvider.kt b/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/sensor/ARCoreDepthProvider.kt new file mode 100644 index 00000000..237ce7b2 --- /dev/null +++ b/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/sensor/ARCoreDepthProvider.kt @@ -0,0 +1,151 @@ +package dev.stapler.stelekit.platform.sensor + +import android.content.Context +import arrow.core.Either +import arrow.core.left +import arrow.core.right +import com.google.ar.core.ArCoreApk +import com.google.ar.core.Config +import com.google.ar.core.Session +import com.google.ar.core.exceptions.CameraNotAvailableException +import com.google.ar.core.exceptions.DeadlineExceededException +import com.google.ar.core.exceptions.NotYetAvailableException +import dev.stapler.stelekit.calibration.DepthFrame +import dev.stapler.stelekit.error.DomainError +import kotlinx.coroutines.CancellationException +import java.nio.ByteOrder + +/** + * ARCore Depth API implementation of [DepthSensorProvider]. + * + * Requires `com.google.ar:core:1.46.0` in androidMain dependencies. + * + * AndroidManifest requirements (already added): + * - `` + * - `` in `` + * + * ADR-005 constraint: when ARCore depth IS active the UI must display: + * "ARCore depth accuracy ±8–10 cm. Not suitable for measurements under 15 cm." + * + * Thread-safety: [acquireDepthFrame] must be called from a single coroutine at a time. + * The ARCore [Session] is not thread-safe; serialise access via a dedicated coroutine context + * if multiple callers are possible. + */ +class ARCoreDepthProvider(private val context: Context) : DepthSensorProvider { + + private var session: Session? = null + + /** + * Returns `true` when ARCore is supported and installed (or can be installed) on this device. + * + * `SUPPORTED_INSTALLED` — AR hardware + Play Services ARCore APK both present. + * `SUPPORTED_NOT_INSTALLED`— AR hardware present but ARCore APK not yet installed; + * the user can install it on demand. + * + * Returns `false` for `UNSUPPORTED_DEVICE_NOT_CAPABLE`, `UNKNOWN_TIMED_OUT`, + * `UNKNOWN_CHECKING`, and `UNKNOWN_ERROR`. + */ + override val isAvailable: Boolean + get() = try { + val availability = ArCoreApk.getInstance().checkAvailability(context) + availability == ArCoreApk.Availability.SUPPORTED_INSTALLED || + availability == ArCoreApk.Availability.SUPPORTED_NOT_INSTALLED + } catch (e: Exception) { + false + } + + /** + * Acquire a single depth frame from ARCore. + * + * Opens an [Session] on the first call (or re-opens after [close]). Configures + * the session for [Config.DepthMode.AUTOMATIC] so depth is captured alongside the + * camera frame without any separate trigger. + * + * Return values: + * - `Right(DepthFrame)` — depth map extracted successfully; depth values are in metres + * (converted from the raw 16-bit millimetre representation). + * - `Right(null)` — ARCore is initialising or does not have enough data yet + * ([NotYetAvailableException] or [DeadlineExceededException]). Not an error. + * - `Left(SensorError.HardwareUnavailable)` — device does not support depth mode. + * - `Left(SensorError.CaptureFailed)` — unexpected ARCore error. + */ + override suspend fun acquireDepthFrame(): Either { + return try { + // Open session lazily on first use. + val arSession = session ?: run { + val s = Session(context) + val config = s.config + config.depthMode = Config.DepthMode.AUTOMATIC + s.configure(config) + session = s + s + } + + if (!arSession.isDepthModeSupported(Config.DepthMode.AUTOMATIC)) { + return DomainError.SensorError.HardwareUnavailable( + "ARCore depth mode not supported on this device" + ).left() + } + + // resume() is idempotent — safe to call even if already resumed. + arSession.resume() + + val frame = arSession.update() + + // Acquire depth image — 16-bit unsigned integer, unit = millimetres. + val depthImage = frame.acquireDepthImage16Bits() + val confidenceImage = frame.acquireRawDepthConfidenceImage() + + val width = depthImage.width + val height = depthImage.height + val pixelCount = width * height + + // Extract depth plane — single plane, pixel stride 2 bytes (uint16 LE). + val depthBuffer = depthImage.planes[0].buffer.order(ByteOrder.LITTLE_ENDIAN) + val depthMapMm = FloatArray(pixelCount) { i -> + // uint16 → unsigned int → metres + (depthBuffer.getShort(i * 2).toInt() and 0xFFFF).toFloat() / 1000f + } + depthImage.close() + + // Extract confidence plane — single plane, pixel stride 1 byte (uint8, range 0–255). + val confBuffer = confidenceImage.planes[0].buffer + val confidenceMap = FloatArray(pixelCount) { i -> + (confBuffer.get(i).toInt() and 0xFF).toFloat() / 255f + } + confidenceImage.close() + + DepthFrame( + width = width, + height = height, + depthMapMm = depthMapMm, + confidenceMap = confidenceMap, + ).right() + } catch (e: NotYetAvailableException) { + // ARCore is still initialising or lighting is insufficient — not an error. + null.right() + } catch (e: DeadlineExceededException) { + // Frame update exceeded the deadline — transient, not an error. + null.right() + } catch (e: CameraNotAvailableException) { + // Camera taken by another app — transient, not an error. + null.right() + } catch (e: Exception) { + if (e is CancellationException) throw e + DomainError.SensorError.CaptureFailed( + "ARCore depth capture failed: ${e.message ?: "unknown"}" + ).left() + } + } + + /** + * Close and release the ARCore session. + * + * Must be called when the host Activity/Fragment stops (e.g. in `onPause` or `onDestroy`) + * to release the camera and GPU resources held by ARCore. + */ + fun close() { + session?.close() + session = null + } +} diff --git a/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/sensor/AndroidCameraProvider.kt b/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/sensor/AndroidCameraProvider.kt new file mode 100644 index 00000000..a9c226a1 --- /dev/null +++ b/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/sensor/AndroidCameraProvider.kt @@ -0,0 +1,183 @@ +package dev.stapler.stelekit.platform.sensor + +import android.Manifest +import android.content.Context +import android.content.pm.PackageManager +import androidx.camera.core.CameraSelector +import androidx.camera.core.ImageCapture +import androidx.camera.core.ImageCaptureException +import androidx.camera.lifecycle.ProcessCameraProvider +import androidx.core.content.ContextCompat +import arrow.core.Either +import arrow.core.left +import arrow.core.right +import dev.stapler.stelekit.error.DomainError +import dev.stapler.stelekit.model.ImageSensorData +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.flow.firstOrNull +import kotlinx.coroutines.suspendCancellableCoroutine +import java.io.File +import java.util.UUID +import java.util.concurrent.Executors +import kotlin.coroutines.resume +import kotlin.coroutines.resumeWithException + +/** + * Android camera provider using CameraX. + * + * Requires the CAMERA runtime permission — checked in [capturePhoto] before binding. + * Uses [ProcessLifecycleOwner] so no Activity reference is needed. + * + * ## OOM prevention (Story 9.2) — inSampleSize for preview loading + * + * Full-resolution JPEG is written to `cacheDir/captures/.jpg`. Display is handled + * by Coil's [coil3.compose.AsyncImage], which subsamples to the viewport resolution + * automatically via [android.graphics.BitmapFactory.Options.inSampleSize]. + * This class never decodes the full bitmap into memory. + * + * ## EXIF correction + * + * After capture, [ExifOrientationFixer.fixOrientation] rotates pixel data to bake in the + * EXIF orientation and resets the orientation tag to NORMAL. Camera metadata + * (focal length, make, model) is extracted from EXIF at that point. + * + * ## Sensor data (Story 8.1.5) + * + * A snapshot of [SensorModule.motionSensorProvider.sensorDataFlow] is captured at shutter + * time and attached to the returned [PlatformImageFile.sensorData]. + */ +class AndroidCameraProvider(private val context: Context) : CameraProvider { + + override val isAvailable: Boolean = true + + /** + * Captures a single JPEG photo using CameraX [ImageCapture]. + * + * Returns [DomainError.SensorError.PermissionDenied] if CAMERA permission is missing. + * Returns [DomainError.SensorError.CaptureFailed] for any capture or I/O error. + */ + @Suppress("TooGenericExceptionCaught") + override suspend fun capturePhoto(): Either { + // 1. Permission gate + if (ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) + != PackageManager.PERMISSION_GRANTED + ) { + return DomainError.SensorError.PermissionDenied("camera").left() + } + + return try { + // 2. Obtain ProcessCameraProvider — bridge ListenableFuture to suspend + val cameraProvider: ProcessCameraProvider = suspendCancellableCoroutine { cont -> + val future = ProcessCameraProvider.getInstance(context) + val executor = ContextCompat.getMainExecutor(context) + future.addListener( + { + if (cont.isActive) { + runCatching { future.get() } + .onSuccess { cont.resume(it) } + .onFailure { cont.resumeWithException(it) } + } + }, + executor, + ) + cont.invokeOnCancellation { future.cancel(true) } + } + + // 3. Build ImageCapture use case + val imageCapture = ImageCapture.Builder() + .setCaptureMode(ImageCapture.CAPTURE_MODE_MAXIMIZE_QUALITY) + .build() + + // 4. Bind to ProcessLifecycleOwner — no Activity reference required. + // Use fully-qualified class to avoid the Glance bindToLifecycle extension clash. + val lifecycleOwner = androidx.lifecycle.ProcessLifecycleOwner.get() + cameraProvider.unbindAll() + cameraProvider.bindToLifecycle( + lifecycleOwner, + CameraSelector.DEFAULT_BACK_CAMERA, + imageCapture, + ) + + // 5. Prepare output file: cacheDir/captures/.jpg + val capturesDir = File(context.cacheDir, "captures").also { it.mkdirs() } + val outputFile = File(capturesDir, "${UUID.randomUUID()}.jpg") + + // 6. Snapshot sensor data at shutter time (Story 8.1.5) + val sensorSnapshot = SensorModule.motionSensorProvider.sensorDataFlow.firstOrNull() + val capturedAt = System.currentTimeMillis() + + // 7. Take the photo — bridge ImageCapture callback to a suspend function + val outputOptions = ImageCapture.OutputFileOptions.Builder(outputFile).build() + val executor = Executors.newSingleThreadExecutor() + + suspendCancellableCoroutine { cont -> + imageCapture.takePicture( + outputOptions, + executor, + object : ImageCapture.OnImageSavedCallback { + override fun onImageSaved(output: ImageCapture.OutputFileResults) { + if (cont.isActive) cont.resume(Unit) + } + + override fun onError(exception: ImageCaptureException) { + if (cont.isActive) cont.resumeWithException(exception) + } + }, + ) + cont.invokeOnCancellation { executor.shutdown() } + } + executor.shutdown() + + if (!outputFile.exists()) { + return DomainError.SensorError.CaptureFailed( + "CameraX onImageSaved fired but file missing: ${outputFile.absolutePath}" + ).left() + } + + // 8. Fix EXIF orientation in-place and extract camera metadata + val fixResult = ExifOrientationFixer.fixOrientation(outputFile.absolutePath) + .fold( + ifLeft = { return it.left() }, + ifRight = { it }, + ) + + // 9. Merge EXIF camera metadata into motion sensor snapshot + val sensorData: ImageSensorData? = if (sensorSnapshot != null) { + sensorSnapshot.copy( + focalLengthMm = fixResult.focalLengthMm ?: sensorSnapshot.focalLengthMm, + focalLength35mmEq = fixResult.focalLength35mmEq + ?: sensorSnapshot.focalLength35mmEq, + cameraMake = fixResult.cameraMake ?: sensorSnapshot.cameraMake, + cameraModel = fixResult.cameraModel ?: sensorSnapshot.cameraModel, + ) + } else if (fixResult.focalLengthMm != null || fixResult.cameraMake != null) { + // No live sensor data — build from EXIF metadata alone + ImageSensorData( + focalLengthMm = fixResult.focalLengthMm, + focalLength35mmEq = fixResult.focalLength35mmEq, + cameraMake = fixResult.cameraMake, + cameraModel = fixResult.cameraModel, + ) + } else { + null + } + + PlatformImageFile( + path = fixResult.outputPath, + mimeType = "image/jpeg", + capturedAtMs = capturedAt, + focalLengthMm = fixResult.focalLengthMm, + focalLength35mmEq = fixResult.focalLength35mmEq, + cameraMake = fixResult.cameraMake, + cameraModel = fixResult.cameraModel, + sensorData = sensorData, + ).right() + } catch (e: CancellationException) { + throw e + } catch (e: Exception) { + DomainError.SensorError.CaptureFailed( + "CameraX capture failed: ${e.message ?: "unknown"}" + ).left() + } + } +} diff --git a/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/sensor/AndroidMotionSensorProvider.kt b/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/sensor/AndroidMotionSensorProvider.kt new file mode 100644 index 00000000..7e596ced --- /dev/null +++ b/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/sensor/AndroidMotionSensorProvider.kt @@ -0,0 +1,227 @@ +package dev.stapler.stelekit.platform.sensor + +import android.Manifest +import android.content.Context +import android.content.pm.PackageManager +import android.hardware.Sensor +import android.hardware.SensorEvent +import android.hardware.SensorEventListener +import android.hardware.SensorManager +import android.location.Location +import android.location.LocationListener +import android.location.LocationManager +import android.os.Bundle +import android.os.Looper +import androidx.core.content.ContextCompat +import dev.stapler.stelekit.model.ImageSensorData +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.filterNotNull + +/** + * Android motion sensor provider using [SensorManager] and the standard [LocationManager]. + * + * Sensors used: + * - [Sensor.TYPE_ROTATION_VECTOR]: device orientation from which pitch, roll, and compass + * bearing (azimuth) are derived via a rotation matrix. + * - [LocationManager] with GPS_PROVIDER at ~1 Hz: provides latLng and altitudeM. + * + * GPS is best-effort: if [Manifest.permission.ACCESS_FINE_LOCATION] is denied, + * GPS fields in emitted [ImageSensorData] will be null but orientation sensors still work. + * + * Sensor listeners and location callbacks are registered on [startSensing] and + * unregistered on [stopSensing] — no battery drain when the editor is not active. + * + * Target emission rate: approximately 10 Hz (SENSOR_DELAY_UI ≈ 66–100 ms on most devices). + * + * Thread safety: [_latestData] is a [MutableStateFlow] — atomic and always holds the most + * recent combined sensor snapshot. Sensor callbacks may arrive on any thread. + */ +class AndroidMotionSensorProvider(private val context: Context) : MotionSensorProvider { + + private val sensorManager = + context.getSystemService(Context.SENSOR_SERVICE) as SensorManager + + private val locationManager = + context.getSystemService(Context.LOCATION_SERVICE) as LocationManager + + // Internal mutable state — combined from all sensor callbacks + private val _latestData = MutableStateFlow(null) + + override val sensorDataFlow: Flow = _latestData.filterNotNull() + + // Current location snapshot — updated by LocationListener callback + @Volatile + private var latLng: Pair? = null + + @Volatile + private var altitudeM: Double? = null + + // Current orientation snapshot — updated by rotation vector callback + @Volatile + private var bearingDeg: Double? = null + + @Volatile + private var pitchDeg: Double? = null + + @Volatile + private var rollDeg: Double? = null + + @Volatile + private var isSensing = false + + // ── Rotation vector sensor listener ────────────────────────────────────── + + private val rotationVectorListener = object : SensorEventListener { + private val rotationMatrix = FloatArray(9) + private val orientationAngles = FloatArray(3) + + override fun onSensorChanged(event: SensorEvent) { + if (event.sensor.type != Sensor.TYPE_ROTATION_VECTOR) return + + // Compute rotation matrix from the rotation vector sensor values + SensorManager.getRotationMatrixFromVector(rotationMatrix, event.values) + + // Get orientation angles: [azimuth, pitch, roll] in radians + // azimuth = orientationAngles[0]: device heading (compass bearing) + // pitch = orientationAngles[1]: front/back tilt (negative = nose up) + // roll = orientationAngles[2]: left/right tilt (positive = right side down) + SensorManager.getOrientation(rotationMatrix, orientationAngles) + + pitchDeg = Math.toDegrees(orientationAngles[1].toDouble()) + rollDeg = Math.toDegrees(orientationAngles[2].toDouble()) + + // Normalize azimuth to [0, 360) + val azimuthDeg = (Math.toDegrees(orientationAngles[0].toDouble()) + 360.0) % 360.0 + bearingDeg = azimuthDeg + + emitCombinedSnapshot() + } + + override fun onAccuracyChanged(sensor: Sensor, accuracy: Int) { + // No action needed — accuracy changes are informational only + } + } + + // ── Location listener ───────────────────────────────────────────────────── + + private val locationListener = object : LocationListener { + override fun onLocationChanged(location: Location) { + latLng = Pair(location.latitude, location.longitude) + altitudeM = if (location.hasAltitude()) location.altitude else null + emitCombinedSnapshot() + } + + @Deprecated("Deprecated in LocationListener") + override fun onStatusChanged(provider: String, status: Int, extras: Bundle) { + // Deprecated in API 29+ — no action needed + } + + override fun onProviderEnabled(provider: String) { + // No action needed + } + + override fun onProviderDisabled(provider: String) { + // GPS was disabled by the user — clear location data + latLng = null + altitudeM = null + emitCombinedSnapshot() + } + } + + // ── MotionSensorProvider implementation ─────────────────────────────────── + + override fun startSensing() { + if (isSensing) return + isSensing = true + + // Register rotation vector sensor (fuses accelerometer + gyroscope + magnetometer + // for the best-quality orientation data on Android) + val rotationSensor = sensorManager.getDefaultSensor(Sensor.TYPE_ROTATION_VECTOR) + if (rotationSensor != null) { + sensorManager.registerListener( + rotationVectorListener, + rotationSensor, + SensorManager.SENSOR_DELAY_UI, // ~60ms ≈ 16 Hz; OS may deliver slower + ) + } + + // Start GPS location updates if ACCESS_FINE_LOCATION permission is granted. + // Graceful fallback: if permission is denied, latLng/altitudeM remain null. + if (hasLocationPermission()) { + startLocationUpdates() + } + + // Emit initial snapshot (all fields may be null until sensors deliver data) + emitCombinedSnapshot() + } + + override fun stopSensing() { + if (!isSensing) return + isSensing = false + + sensorManager.unregisterListener(rotationVectorListener) + + // Unconditionally attempt to remove location updates — safe even if + // they were never registered (LocationManager ignores unknown listeners) + try { + locationManager.removeUpdates(locationListener) + } catch (_: Exception) { + // SecurityException can occur on some devices if permission was revoked + // between startSensing() and stopSensing(). Safe to swallow. + } + } + + // ── Helpers ─────────────────────────────────────────────────────────────── + + private fun hasLocationPermission(): Boolean = + ContextCompat.checkSelfPermission( + context, + Manifest.permission.ACCESS_FINE_LOCATION, + ) == PackageManager.PERMISSION_GRANTED + + @Suppress("MissingPermission") // Permission is checked in hasLocationPermission() before call + private fun startLocationUpdates() { + try { + val isGpsEnabled = locationManager.isProviderEnabled(LocationManager.GPS_PROVIDER) + if (isGpsEnabled) { + locationManager.requestLocationUpdates( + LocationManager.GPS_PROVIDER, + 1_000L, // minTimeMs: 1 Hz + 0f, // minDistanceMeters: emit on every update regardless of movement + locationListener, + Looper.getMainLooper(), + ) + } + // Also request NETWORK provider as a lower-accuracy fallback (works indoors) + val isNetworkEnabled = locationManager.isProviderEnabled(LocationManager.NETWORK_PROVIDER) + if (isNetworkEnabled) { + locationManager.requestLocationUpdates( + LocationManager.NETWORK_PROVIDER, + 2_000L, // 0.5 Hz — network location drains less battery than GPS + 0f, + locationListener, + Looper.getMainLooper(), + ) + } + } catch (_: SecurityException) { + // Permission was revoked between hasLocationPermission() and this call. + // GPS fields will remain null — safe to continue without location. + } + } + + /** + * Combine all sensor snapshots into a single [ImageSensorData] and emit to [sensorDataFlow]. + * + * Called from sensor/location callbacks — must be fast; no blocking IO here. + */ + private fun emitCombinedSnapshot() { + _latestData.value = ImageSensorData( + latLng = latLng, + altitudeM = altitudeM, + bearingDeg = bearingDeg, + pitchDeg = pitchDeg, + rollDeg = rollDeg, + ) + } +} diff --git a/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/sensor/AndroidPhotoPickerLauncher.kt b/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/sensor/AndroidPhotoPickerLauncher.kt new file mode 100644 index 00000000..04c9e782 --- /dev/null +++ b/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/sensor/AndroidPhotoPickerLauncher.kt @@ -0,0 +1,132 @@ +package dev.stapler.stelekit.platform.sensor + +import android.content.Context +import android.net.Uri +import androidx.activity.result.ActivityResultLauncher +import androidx.activity.result.PickVisualMediaRequest +import androidx.activity.result.contract.ActivityResultContracts +import arrow.core.Either +import arrow.core.left +import arrow.core.right +import dev.stapler.stelekit.error.DomainError +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CompletableDeferred +import java.io.File +import java.io.IOException + +/** + * Launches the Android Photo Picker (API 33+ native; backported to API 21+ via Play Services) + * and copies the selected image into a stable local temp file before handing control back + * to the caller. + * + * CRITICAL — content URI expiry: + * The content URI returned by the Photo Picker carries a temporary read grant that expires + * as soon as the process yields control across a suspension point. [readBytesFromUri] MUST be + * called synchronously inside the [ActivityResultCallback] — BEFORE any `withContext` or `await`. + * + * Usage pattern (Activity/Fragment): + * ```kotlin + * val launcher = registerForActivityResult(ActivityResultContracts.PickVisualMedia()) { uri -> + * uri ?: return@registerForActivityResult + * // READ URI BYTES HERE — before any coroutine suspension + * val bytes = contentResolver.openInputStream(uri)?.use { it.readBytes() } + * // Then hand bytes to ImageImportService on a background coroutine + * } + * ``` + * + * This class wraps that pattern with a coroutine-friendly API via [CompletableDeferred]. + * It must be registered once per Activity (not per composable) because + * [ActivityResultLauncher] must be registered before onStart. + */ +class AndroidPhotoPickerLauncher( + private val context: Context, +) { + private var pendingResult: CompletableDeferred>? = null + private var launcher: ActivityResultLauncher? = null + + /** + * Register the Photo Picker result contract. + * + * Must be called from [androidx.activity.ComponentActivity] or a Fragment during + * `onCreate` — before `onStart`. + * + * @param registerForActivityResult Use `registerForActivityResult` from the Activity. + */ + fun register( + registerForActivityResult: ( + ActivityResultContracts.PickVisualMedia, + (Uri?) -> Unit, + ) -> ActivityResultLauncher, + ) { + launcher = registerForActivityResult(ActivityResultContracts.PickVisualMedia()) { uri -> + val deferred = pendingResult + if (deferred == null || deferred.isCompleted) return@registerForActivityResult + + if (uri == null) { + deferred.complete( + DomainError.SensorError.CaptureFailed("User cancelled photo picker").left() + ) + return@registerForActivityResult + } + + // CRITICAL: Read URI bytes HERE, synchronously, before any coroutine suspension. + // The content URI grant from the Photo Picker is temporary — it expires when + // this callback returns. Calling openInputStream on a background dispatcher + // would observe an expired grant and throw SecurityException. + val result = readBytesFromUri(uri) + deferred.complete(result) + } + } + + /** + * Launch the Photo Picker and suspend until the user makes a selection or cancels. + * + * Returns a [PlatformImageFile] whose [PlatformImageFile.path] points to a stable + * temp file inside the app's cache directory — safe to pass to [ImageImportService]. + */ + suspend fun pickPhoto(): Either { + val l = launcher ?: return DomainError.SensorError.HardwareUnavailable( + "PhotoPickerLauncher not registered — call register() in Activity.onCreate" + ).left() + + val deferred = CompletableDeferred>() + pendingResult = deferred + + l.launch(PickVisualMediaRequest(ActivityResultContracts.PickVisualMedia.ImageOnly)) + return deferred.await() + } + + /** + * Read all bytes from [uri] synchronously via ContentResolver and write them to a + * temp file in the app's cache directory. + * + * This MUST run on the thread that called the [ActivityResultCallback] — before + * any coroutine suspension boundary. + */ + private fun readBytesFromUri(uri: Uri): Either { + return try { + val bytes = context.contentResolver.openInputStream(uri) + ?.use { it.readBytes() } + ?: return DomainError.SensorError.CaptureFailed( + "contentResolver.openInputStream returned null for $uri" + ).left() + + val tempFile = File(context.cacheDir, "photo_import_${System.currentTimeMillis()}.jpg") + tempFile.writeBytes(bytes) + + PlatformImageFile( + path = tempFile.absolutePath, + mimeType = context.contentResolver.getType(uri) ?: "image/jpeg", + capturedAtMs = System.currentTimeMillis(), + ).right() + } catch (e: CancellationException) { + throw e + } catch (e: SecurityException) { + DomainError.SensorError.PermissionDenied("photo_picker: ${e.message}").left() + } catch (e: IOException) { + DomainError.SensorError.CaptureFailed("I/O error reading photo picker URI: ${e.message}").left() + } catch (e: Exception) { + DomainError.SensorError.CaptureFailed("Unexpected error: ${e.message}").left() + } + } +} diff --git a/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/sensor/ExifOrientationFixer.kt b/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/sensor/ExifOrientationFixer.kt new file mode 100644 index 00000000..da99a08f --- /dev/null +++ b/kmp/src/androidMain/kotlin/dev/stapler/stelekit/platform/sensor/ExifOrientationFixer.kt @@ -0,0 +1,174 @@ +package dev.stapler.stelekit.platform.sensor + +import android.graphics.Bitmap +import android.graphics.BitmapFactory +import android.graphics.Matrix +import androidx.exifinterface.media.ExifInterface +import arrow.core.Either +import arrow.core.left +import arrow.core.right +import dev.stapler.stelekit.error.DomainError +import kotlinx.coroutines.CancellationException +import java.io.File +import java.io.FileOutputStream + +/** + * Corrects EXIF orientation tags in JPEG files captured on Android. + * + * Samsung and many other Android OEMs write JPEG data in sensor-native orientation + * and set the EXIF `Orientation` tag to describe the rotation needed for correct display. + * Many downstream consumers (including SteleKit's annotation canvas) do not honour EXIF + * tags — this fixer bakes the rotation into the pixel data and resets the tag to NORMAL. + * + * Also extracts calibration-relevant EXIF fields ([focalLengthMm], [focalLength35mmEq], + * [cameraMake], [cameraModel]) for use in [dev.stapler.stelekit.model.ImageSensorData]. + * + * Requires `androidx.exifinterface:exifinterface` on the classpath (already present via + * the `androidx.appcompat:appcompat` transitive dependency on API 21+ targets, but + * explicitly declared for clarity). + */ +object ExifOrientationFixer { + + /** + * Result of [fixOrientation]. + * + * [outputPath] is the file path of the corrected JPEG (may equal [inputPath] when + * overwriting in-place). [sensorData] contains EXIF-extracted camera metadata. + */ + data class FixResult( + val outputPath: String, + val focalLengthMm: Double?, + val focalLength35mmEq: Double?, + val cameraMake: String?, + val cameraModel: String?, + ) + + /** + * Read the EXIF orientation from [inputPath], rotate the decoded [Bitmap] if needed, + * and write the corrected JPEG to [outputPath]. + * + * If [outputPath] is null, the corrected image is written to [inputPath] in-place. + * + * Returns [Either.Right] with [FixResult] on success. + * Returns [Either.Left] with a [DomainError.SensorError.CaptureFailed] on I/O or + * decoding failure. + */ + fun fixOrientation( + inputPath: String, + outputPath: String? = null, + jpegQuality: Int = 95, + ): Either { + return try { + val exif = ExifInterface(inputPath) + + // Extract calibration-relevant EXIF fields + val focalLengthMm = exif.getAttribute(ExifInterface.TAG_FOCAL_LENGTH) + ?.let { parseRational(it) } + val focalLength35mmEq = exif.getAttributeInt( + ExifInterface.TAG_FOCAL_LENGTH_IN_35MM_FILM, 0 + ).takeIf { it > 0 }?.toDouble() + val cameraMake = exif.getAttribute(ExifInterface.TAG_MAKE)?.takeIf { it.isNotBlank() } + val cameraModel = exif.getAttribute(ExifInterface.TAG_MODEL)?.takeIf { it.isNotBlank() } + + val orientationValue = exif.getAttributeInt( + ExifInterface.TAG_ORIENTATION, + ExifInterface.ORIENTATION_NORMAL + ) + + val matrix = buildRotationMatrix(orientationValue) + val destination = outputPath ?: inputPath + + if (matrix == null) { + // No rotation needed — just copy if paths differ, then return result. + if (outputPath != null && outputPath != inputPath) { + File(inputPath).copyTo(File(outputPath), overwrite = true) + } + } else { + // Decode, rotate, re-encode + val original = BitmapFactory.decodeFile(inputPath) + ?: return DomainError.SensorError.CaptureFailed( + "BitmapFactory.decodeFile returned null for $inputPath" + ).left() + val rotated = Bitmap.createBitmap( + original, 0, 0, original.width, original.height, matrix, true + ) + original.recycle() + + FileOutputStream(destination).use { out -> + rotated.compress(Bitmap.CompressFormat.JPEG, jpegQuality, out) + } + rotated.recycle() + + // Reset orientation tag to NORMAL in the output file + val outExif = ExifInterface(destination) + outExif.setAttribute( + ExifInterface.TAG_ORIENTATION, + ExifInterface.ORIENTATION_NORMAL.toString() + ) + outExif.saveAttributes() + } + + FixResult( + outputPath = destination, + focalLengthMm = focalLengthMm, + focalLength35mmEq = focalLength35mmEq, + cameraMake = cameraMake, + cameraModel = cameraModel, + ).right() + } catch (e: CancellationException) { + throw e + } catch (e: Exception) { + DomainError.SensorError.CaptureFailed( + "ExifOrientationFixer failed for $inputPath: ${e.message ?: "unknown"}" + ).left() + } + } + + /** + * Build a [Matrix] that applies the rotation/flip described by [orientationValue]. + * + * Returns `null` when no transformation is needed (ORIENTATION_NORMAL or unknown). + */ + private fun buildRotationMatrix(orientationValue: Int): Matrix? { + val matrix = Matrix() + var needsTransform = true + when (orientationValue) { + ExifInterface.ORIENTATION_ROTATE_90 -> matrix.postRotate(90f) + ExifInterface.ORIENTATION_ROTATE_180 -> matrix.postRotate(180f) + ExifInterface.ORIENTATION_ROTATE_270 -> matrix.postRotate(270f) + ExifInterface.ORIENTATION_FLIP_HORIZONTAL -> matrix.postScale(-1f, 1f) + ExifInterface.ORIENTATION_FLIP_VERTICAL -> matrix.postScale(1f, -1f) + ExifInterface.ORIENTATION_TRANSPOSE -> { + matrix.postRotate(90f) + matrix.postScale(-1f, 1f) + } + ExifInterface.ORIENTATION_TRANSVERSE -> { + matrix.postRotate(270f) + matrix.postScale(-1f, 1f) + } + else -> needsTransform = false + } + return if (needsTransform) matrix else null + } + + /** + * Parse a rational EXIF string like "3670/1000" or "3.67" to a [Double]. + * Returns null on failure. + */ + private fun parseRational(value: String): Double? { + return try { + if (value.contains('/')) { + val parts = value.split('/') + if (parts.size == 2) { + val num = parts[0].trim().toDoubleOrNull() ?: return null + val den = parts[1].trim().toDoubleOrNull() ?: return null + if (den == 0.0) null else num / den + } else null + } else { + value.toDoubleOrNull() + } + } catch (_: Exception) { + null + } + } +} diff --git a/kmp/src/androidMain/kotlin/dev/stapler/stelekit/ui/annotate/ImageEncoder.android.kt b/kmp/src/androidMain/kotlin/dev/stapler/stelekit/ui/annotate/ImageEncoder.android.kt new file mode 100644 index 00000000..8da77ea4 --- /dev/null +++ b/kmp/src/androidMain/kotlin/dev/stapler/stelekit/ui/annotate/ImageEncoder.android.kt @@ -0,0 +1,22 @@ +package dev.stapler.stelekit.ui.annotate + +import android.graphics.Bitmap +import androidx.compose.ui.graphics.ImageBitmap +import androidx.compose.ui.graphics.asAndroidBitmap +import java.io.ByteArrayOutputStream + +/** + * Android JPEG encoder using [Bitmap.compress]. + */ +actual object ImageEncoder { + actual fun encodeToJpeg(bitmap: ImageBitmap, quality: Int): ByteArray { + return try { + val androidBitmap: Bitmap = bitmap.asAndroidBitmap() + val out = ByteArrayOutputStream() + androidBitmap.compress(Bitmap.CompressFormat.JPEG, quality.coerceIn(0, 100), out) + out.toByteArray() + } catch (e: Exception) { + ByteArray(0) + } + } +} diff --git a/kmp/src/androidUnitTest/kotlin/dev/stapler/stelekit/platform/sensor/ExifOrientationFixerTest.kt b/kmp/src/androidUnitTest/kotlin/dev/stapler/stelekit/platform/sensor/ExifOrientationFixerTest.kt new file mode 100644 index 00000000..b2c62301 --- /dev/null +++ b/kmp/src/androidUnitTest/kotlin/dev/stapler/stelekit/platform/sensor/ExifOrientationFixerTest.kt @@ -0,0 +1,288 @@ +package dev.stapler.stelekit.platform.sensor + +import android.graphics.Bitmap +import androidx.exifinterface.media.ExifInterface +import arrow.core.Either +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config +import java.io.File +import java.io.FileOutputStream +import kotlin.test.assertEquals +import kotlin.test.assertIs +import kotlin.test.assertNull + +/** + * Regression tests for [ExifOrientationFixer] verifying that Samsung-style EXIF + * orientation tags (written at capture time) are baked into pixel data and reset to NORMAL. + * + * Uses Robolectric to run Android Bitmap / ExifInterface APIs on the JVM. + * + * Orientations tested: + * - 1 (NORMAL) — no rotation applied, file copied/left unchanged + * - 3 (ROTATE 180) — bitmap rotated 180° + * - 6 (ROTATE 90) — typical Samsung portrait-mode capture, rotated 90° CW + * - 8 (ROTATE 270) — landscape-flipped capture, rotated 90° CCW + */ +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [34]) +class ExifOrientationFixerTest { + + @get:Rule + val tempFolder = TemporaryFolder() + + // ── Helpers ─────────────────────────────────────────────────────────────── + + /** + * Write a minimal 10×10 JPEG to [file] and inject the given EXIF [orientation]. + * + * Uses [Bitmap.compress] (Robolectric provides a shadow that produces a real JPEG + * in the temp filesystem) then sets the orientation tag via [ExifInterface]. + */ + private fun writeJpegWithOrientation(file: File, orientation: Int) { + // Create a 10×10 RGB bitmap — small enough that decoding is fast. + val bitmap = Bitmap.createBitmap(10, 10, Bitmap.Config.ARGB_8888) + FileOutputStream(file).use { out -> + bitmap.compress(Bitmap.CompressFormat.JPEG, 95, out) + } + bitmap.recycle() + + // Inject the desired EXIF orientation tag. + val exif = ExifInterface(file.absolutePath) + exif.setAttribute(ExifInterface.TAG_ORIENTATION, orientation.toString()) + exif.saveAttributes() + } + + /** Read the EXIF orientation tag from [file]. Returns the integer value. */ + private fun readOrientation(file: File): Int { + val exif = ExifInterface(file.absolutePath) + return exif.getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL) + } + + // ── Orientation 1 (NORMAL): no transformation needed ───────────────────── + + @Test + fun `orientation NORMAL returns success and leaves file unchanged`() { + val input = tempFolder.newFile("normal.jpg") + writeJpegWithOrientation(input, ExifInterface.ORIENTATION_NORMAL) + + val result = ExifOrientationFixer.fixOrientation(input.absolutePath) + + assertIs>(result) + assertEquals(input.absolutePath, result.value.outputPath) + // Tag should still be NORMAL (or absent) after a no-op pass. + val orientationAfter = readOrientation(input) + assertEquals(ExifInterface.ORIENTATION_NORMAL, orientationAfter) + } + + // ── Orientation 3 (ROTATE_180): 180° rotation ──────────────────────────── + + @Test + fun `orientation ROTATE_180 fixes file and resets tag to NORMAL`() { + val input = tempFolder.newFile("rotate180.jpg") + writeJpegWithOrientation(input, ExifInterface.ORIENTATION_ROTATE_180) + + val result = ExifOrientationFixer.fixOrientation(input.absolutePath) + + assertIs>(result) + // After fixing, the tag must be reset to NORMAL (= 1). + val orientationAfter = readOrientation(File(result.value.outputPath)) + assertEquals(ExifInterface.ORIENTATION_NORMAL, orientationAfter, + "Orientation tag must be reset to NORMAL after rotation correction") + } + + // ── Orientation 6 (ROTATE_90): Samsung portrait capture ────────────────── + + @Test + fun `orientation ROTATE_90 fixes file and resets tag to NORMAL`() { + val input = tempFolder.newFile("rotate90.jpg") + writeJpegWithOrientation(input, ExifInterface.ORIENTATION_ROTATE_90) + + val result = ExifOrientationFixer.fixOrientation(input.absolutePath) + + assertIs>(result) + val orientationAfter = readOrientation(File(result.value.outputPath)) + assertEquals(ExifInterface.ORIENTATION_NORMAL, orientationAfter, + "Orientation tag must be reset to NORMAL after 90° CW correction") + } + + // ── Orientation 8 (ROTATE_270): 90° CCW / landscape-flipped capture ────── + + @Test + fun `orientation ROTATE_270 fixes file and resets tag to NORMAL`() { + val input = tempFolder.newFile("rotate270.jpg") + writeJpegWithOrientation(input, ExifInterface.ORIENTATION_ROTATE_270) + + val result = ExifOrientationFixer.fixOrientation(input.absolutePath) + + assertIs>(result) + val orientationAfter = readOrientation(File(result.value.outputPath)) + assertEquals(ExifInterface.ORIENTATION_NORMAL, orientationAfter, + "Orientation tag must be reset to NORMAL after 90° CCW correction") + } + + // ── Output path override ────────────────────────────────────────────────── + + @Test + fun `explicit outputPath writes corrected image to separate file`() { + val input = tempFolder.newFile("input.jpg") + val output = tempFolder.newFile("output.jpg") + writeJpegWithOrientation(input, ExifInterface.ORIENTATION_ROTATE_90) + + val result = ExifOrientationFixer.fixOrientation(input.absolutePath, output.absolutePath) + + assertIs>(result) + assertEquals(output.absolutePath, result.value.outputPath) + // Original file should be untouched. + assertEquals(ExifInterface.ORIENTATION_ROTATE_90, readOrientation(input)) + // Output file should have NORMAL orientation. + assertEquals(ExifInterface.ORIENTATION_NORMAL, readOrientation(output)) + } + + // ── EXIF metadata extraction ────────────────────────────────────────────── + + @Test + fun `fixOrientation extracts focal length when present`() { + val input = tempFolder.newFile("focal.jpg") + writeJpegWithOrientation(input, ExifInterface.ORIENTATION_NORMAL) + + // Inject a focal length rational value (e.g. 4.25mm expressed as "4250/1000"). + val exif = ExifInterface(input.absolutePath) + exif.setAttribute(ExifInterface.TAG_FOCAL_LENGTH, "4250/1000") + exif.saveAttributes() + + val result = ExifOrientationFixer.fixOrientation(input.absolutePath) + + assertIs>(result) + val focal = result.value.focalLengthMm + assert(focal != null && focal > 4.0 && focal < 5.0) { + "Expected focal length ~4.25mm but got $focal" + } + } + + @Test + fun `fixOrientation returns null focal length when EXIF tag absent`() { + val input = tempFolder.newFile("nofocal.jpg") + writeJpegWithOrientation(input, ExifInterface.ORIENTATION_NORMAL) + // No focal length tag injected. + + val result = ExifOrientationFixer.fixOrientation(input.absolutePath) + + assertIs>(result) + assertNull(result.value.focalLengthMm) + } + + // ── Orientation 7 (TRANSVERSE): Samsung edge case — 270° + horizontal flip ─ + + /** + * ORIENTATION_TRANSVERSE (value = 7) is the Samsung edge-case orientation: + * rotate 270° then flip horizontally. This must produce a corrected JPEG + * with the orientation tag reset to NORMAL. + * + * Regression test for the Samsung Galaxy capture bug where TRANSVERSE images + * appear mirrored and sideways in consumers that ignore EXIF tags. + */ + @Test + fun `orientation TRANSVERSE (Samsung edge case) fixes file and resets tag to NORMAL`() { + val input = tempFolder.newFile("transverse.jpg") + writeJpegWithOrientation(input, ExifInterface.ORIENTATION_TRANSVERSE) + + val result = ExifOrientationFixer.fixOrientation(input.absolutePath) + + assertIs>(result) + val orientationAfter = readOrientation(File(result.value.outputPath)) + assertEquals( + ExifInterface.ORIENTATION_NORMAL, orientationAfter, + "ORIENTATION_TRANSVERSE must be baked in and tag reset to NORMAL" + ) + } + + // ── Landscape capture with incorrect ROTATE_90 tag ──────────────────────── + + /** + * Simulates a landscape photo (wider than tall) that has an incorrect ROTATE_90 + * EXIF orientation tag — common on some Android OEMs that write the sensor-native + * portrait orientation even for landscape captures. + * + * After fixing, the output file must have ORIENTATION_NORMAL so that downstream + * consumers (annotation canvas, Coil) display the image correctly. + */ + @Test + fun `landscape image with incorrect ROTATE_90 tag is corrected and tag reset`() { + // Create a landscape bitmap: wider (20px) than tall (10px). + val bitmap = Bitmap.createBitmap(20, 10, Bitmap.Config.ARGB_8888) + val input = tempFolder.newFile("landscape_wrong_tag.jpg") + FileOutputStream(input).use { out -> + bitmap.compress(Bitmap.CompressFormat.JPEG, 95, out) + } + bitmap.recycle() + + // Tag incorrectly as ROTATE_90 (portrait capture tag applied to a landscape file). + val exif = ExifInterface(input.absolutePath) + exif.setAttribute(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_ROTATE_90.toString()) + exif.saveAttributes() + + val result = ExifOrientationFixer.fixOrientation(input.absolutePath) + + assertIs>(result) + val orientationAfter = readOrientation(File(result.value.outputPath)) + assertEquals( + ExifInterface.ORIENTATION_NORMAL, orientationAfter, + "Landscape image with incorrect ROTATE_90 tag must be corrected to NORMAL" + ) + } + + // ── Google Pixel portrait: ORIENTATION_NORMAL — no spurious rotation ────── + + /** + * Google Pixel phones capture portrait images with ORIENTATION_NORMAL (tag = 1) + * because the sensor is mounted in portrait orientation. No rotation must be + * applied; the pixel data must not be modified. + * + * Regression test ensuring [ExifOrientationFixer] never spuriously rotates a + * file that is already correctly oriented. + */ + @Test + fun `Google Pixel portrait (ORIENTATION_NORMAL) is not spuriously rotated`() { + // Create a portrait bitmap: taller (20px) than wide (10px). + val bitmap = Bitmap.createBitmap(10, 20, Bitmap.Config.ARGB_8888) + val input = tempFolder.newFile("pixel_portrait.jpg") + FileOutputStream(input).use { out -> + bitmap.compress(Bitmap.CompressFormat.JPEG, 95, out) + } + bitmap.recycle() + + // Pixel sets ORIENTATION_NORMAL — correct as-is. + val exif = ExifInterface(input.absolutePath) + exif.setAttribute(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL.toString()) + exif.saveAttributes() + + val originalSize = input.length() + + val result = ExifOrientationFixer.fixOrientation(input.absolutePath) + + assertIs>(result) + // Tag must remain NORMAL. + val orientationAfter = readOrientation(File(result.value.outputPath)) + assertEquals( + ExifInterface.ORIENTATION_NORMAL, orientationAfter, + "Google Pixel portrait with ORIENTATION_NORMAL must not be rotated" + ) + // The file should not have been re-encoded (in-place no-op path). + assertEquals( + originalSize, input.length(), + "No-op path must not re-encode the file (file size must be unchanged)" + ) + } + + // ── Error handling ──────────────────────────────────────────────────────── + + @Test + fun `fixOrientation returns CaptureFailed for non-existent file`() { + val result = ExifOrientationFixer.fixOrientation("/tmp/does_not_exist_abc123.jpg") + assertIs>(result) + } +} diff --git a/kmp/src/businessTest/kotlin/dev/stapler/stelekit/ui/GalleryViewModelTest.kt b/kmp/src/businessTest/kotlin/dev/stapler/stelekit/ui/GalleryViewModelTest.kt new file mode 100644 index 00000000..b9d8a76d --- /dev/null +++ b/kmp/src/businessTest/kotlin/dev/stapler/stelekit/ui/GalleryViewModelTest.kt @@ -0,0 +1,155 @@ +// Copyright (c) 2026 Tyler Stapler +// SPDX-License-Identifier: Elastic-2.0 + +package dev.stapler.stelekit.ui + +import dev.stapler.stelekit.model.Calibration +import dev.stapler.stelekit.model.CalibrationMethod +import dev.stapler.stelekit.model.ImageAnnotation +import dev.stapler.stelekit.model.MeasurementUnit +import dev.stapler.stelekit.repository.InMemoryImageAnnotationRepository +import dev.stapler.stelekit.ui.gallery.GallerySortOrder +import dev.stapler.stelekit.ui.gallery.GalleryViewModel +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.runBlocking +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNull +import kotlin.test.assertTrue + +class GalleryViewModelTest { + + private fun makeRepo(vararg annotations: ImageAnnotation): InMemoryImageAnnotationRepository { + val repo = InMemoryImageAnnotationRepository() + annotations.forEach { repo.upsert(it) } + return repo + } + + private fun makeImage( + uuid: String, + tags: List = emptyList(), + importedAtMs: Long = 0L, + capturedAtMs: Long? = null, + ): ImageAnnotation = ImageAnnotation( + uuid = uuid, + blockUuid = "blk-$uuid", + pageUuid = "page-1", + graphPath = "/graphs/test", + filePath = "/graphs/test/assets/images/$uuid.jpg", + tags = tags, + importedAtMs = importedAtMs, + capturedAtMs = capturedAtMs, + ) + + // ── 1. Initial load exposes all images ──────────────────────────────────── + + @Test + fun initialLoad_exposesAllImages() = runBlocking { + val img1 = makeImage("img-1", importedAtMs = 100) + val img2 = makeImage("img-2", importedAtMs = 200) + val repo = makeRepo(img1, img2) + val vm = GalleryViewModel(repo) + + // Allow the first collection to settle + val state = vm.state.first { !it.isLoading } + assertEquals(2, state.images.size, "Expected 2 images in gallery") + vm.close() + } + + // ── 2. selectTag filters to matching annotations ────────────────────────── + + @Test + fun selectTag_filtersImages() = runBlocking { + val img1 = makeImage("img-1", tags = listOf("kitchen")) + val img2 = makeImage("img-2", tags = listOf("bathroom")) + val img3 = makeImage("img-3", tags = listOf("kitchen", "bathroom")) + val repo = makeRepo(img1, img2, img3) + val vm = GalleryViewModel(repo) + + vm.selectTag("kitchen") + + val state = vm.state.first { !it.isLoading } + val filtered = state.images + assertEquals(2, filtered.size, "Expected 2 images tagged 'kitchen'") + assertTrue(filtered.any { it.uuid == "img-1" }) + assertTrue(filtered.any { it.uuid == "img-3" }) + assertEquals("kitchen", state.selectedTag) + vm.close() + } + + // ── 3. selectTag(null) removes filter ───────────────────────────────────── + + @Test + fun selectTag_null_clearsFilter() = runBlocking { + val img1 = makeImage("img-1", tags = listOf("kitchen")) + val img2 = makeImage("img-2", tags = listOf("bathroom")) + val repo = makeRepo(img1, img2) + val vm = GalleryViewModel(repo) + + vm.selectTag("kitchen") + vm.selectTag(null) + + val state = vm.state.first { !it.isLoading } + assertEquals(2, state.images.size, "Expected all images after clearing filter") + assertNull(state.selectedTag) + vm.close() + } + + // ── 4. Sort BY_DATE_IMPORTED orders newest-first ────────────────────────── + + @Test + fun sortByImportDate_newestFirst() = runBlocking { + val img1 = makeImage("img-1", importedAtMs = 100) + val img2 = makeImage("img-2", importedAtMs = 300) + val img3 = makeImage("img-3", importedAtMs = 200) + val repo = makeRepo(img1, img2, img3) + val vm = GalleryViewModel(repo) + + vm.setSortOrder(GallerySortOrder.BY_DATE_IMPORTED) + + val state = vm.state.first { !it.isLoading } + val uuids = state.images.map { it.uuid } + assertEquals(listOf("img-2", "img-3", "img-1"), uuids, "Expected newest-import first") + vm.close() + } + + // ── 5. Sort BY_DATE_CAPTURED uses capturedAtMs ──────────────────────────── + + @Test + fun sortByCaptureDate_newestFirst() = runBlocking { + val img1 = makeImage("img-1", capturedAtMs = 50) + val img2 = makeImage("img-2", capturedAtMs = 200) + val img3 = makeImage("img-3", capturedAtMs = null) // no capture date + val repo = makeRepo(img1, img2, img3) + val vm = GalleryViewModel(repo) + + vm.setSortOrder(GallerySortOrder.BY_DATE_CAPTURED) + + val state = vm.state.first { !it.isLoading } + // Images with capture date come first (newest first), then null-capture images + val uuids = state.images.map { it.uuid } + assertTrue(uuids.indexOf("img-2") < uuids.indexOf("img-1"), "img-2 (newer) should precede img-1") + vm.close() + } + + // ── 6. Default sort order is BY_DATE_IMPORTED ───────────────────────────── + + @Test + fun defaultSortOrder_isByDateImported() = runBlocking { + val repo = makeRepo() + val vm = GalleryViewModel(repo) + val state = vm.state.first() + assertEquals(GallerySortOrder.BY_DATE_IMPORTED, state.sortOrder) + vm.close() + } + + // ── 7. Empty repo produces empty state ──────────────────────────────────── + + @Test + fun emptyRepo_emptyState() = runBlocking { + val vm = GalleryViewModel(InMemoryImageAnnotationRepository()) + val state = vm.state.first { !it.isLoading } + assertTrue(state.images.isEmpty()) + vm.close() + } +} diff --git a/kmp/src/commonMain/kotlin/dev/stapler/stelekit/calibration/CalibrationFallbackChain.kt b/kmp/src/commonMain/kotlin/dev/stapler/stelekit/calibration/CalibrationFallbackChain.kt new file mode 100644 index 00000000..7ded50ab --- /dev/null +++ b/kmp/src/commonMain/kotlin/dev/stapler/stelekit/calibration/CalibrationFallbackChain.kt @@ -0,0 +1,139 @@ +package dev.stapler.stelekit.calibration + +import dev.stapler.stelekit.logging.Logger +import dev.stapler.stelekit.model.Calibration +import dev.stapler.stelekit.model.CalibrationMethod +import dev.stapler.stelekit.model.ImageSensorData +import dev.stapler.stelekit.platform.ml.MonocularDepthEstimator +import dev.stapler.stelekit.platform.sensor.DepthSensorProvider +import dev.stapler.stelekit.model.NormalizedPoint + +/** + * Calibration fallback chain. + * + * Tries calibration sources in descending accuracy order: + * 1. BLE laser reading (injected externally — caller provides [bleCalibration]) + * 2. Manual reference (injected externally — caller provides [manualCalibration]) + * 3. ARCore / LiDAR depth (via [depthSensorProvider], if [isAvailable]) + * 4. EXIF focal-length math (via [ExifCalibrationService], if [ImageSensorData] has focal data) + * 5. Monocular ML depth (via [monocularDepthEstimator], if [isAvailable]) + * 6. [CalibrationMethod.NONE] — no calibration available + * + * Each skipped step is logged at INFO level with the reason. + * + * @param depthSensorProvider depth hardware abstraction (ARCore/LiDAR/NoOp) + * @param monocularDepthEstimator ML depth estimator (ONNX/CoreML/NoOp) + */ +class CalibrationFallbackChain( + private val depthSensorProvider: DepthSensorProvider, + private val monocularDepthEstimator: MonocularDepthEstimator, +) { + private val logger = Logger("CalibrationFallbackChain") + + /** + * Determine the best available [Calibration] using all available sources. + * + * @param bleCalibration non-null if a BLE laser reading was injected + * @param manualCalibration non-null if the user has drawn a reference line + * @param sensorData EXIF sensor data for focal-length estimation + * @param imageWidthPx native image width (for EXIF math) + * @param depthTapPoint normalized tap point for depth-based calibration + * @param mlDepthMap depth map from ML estimator (may be null) + * @param depthHintMeters optional depth hint for EXIF estimation + * @return the best [Calibration] available, or a [CalibrationMethod.NONE] calibration + */ + suspend fun resolve( + bleCalibration: Calibration? = null, + manualCalibration: Calibration? = null, + sensorData: ImageSensorData? = null, + imageWidthPx: Double = 0.0, + depthTapPoint: NormalizedPoint? = null, + mlDepthMap: FloatArray? = null, + imageHeightPx: Double = 0.0, + depthHintMeters: Double? = null, + ): Calibration { + + // 1. BLE laser — highest accuracy (±1 mm) + if (bleCalibration != null) { + logger.info("CalibrationFallbackChain: using BLE_LASER calibration (±1mm)") + return bleCalibration + } + logger.info("CalibrationFallbackChain: BLE_LASER not available — no laser reading injected") + + // 2. Manual reference object — 100% confidence by definition + if (manualCalibration != null) { + logger.info("CalibrationFallbackChain: using MANUAL_REFERENCE calibration (100% confidence)") + return manualCalibration + } + logger.info("CalibrationFallbackChain: MANUAL_REFERENCE not available — no reference line drawn") + + // 3. ARCore / LiDAR depth + if (depthSensorProvider.isAvailable && depthTapPoint != null && imageWidthPx > 0.0) { + val frameResult = depthSensorProvider.acquireDepthFrame() + frameResult.fold( + ifLeft = { err -> + logger.info("CalibrationFallbackChain: depth sensor skipped — ${err.message}") + }, + ifRight = { frame -> + if (frame != null) { + val cal = CalibrationService.computeFromDepthFrame(frame, depthTapPoint, imageWidthPx) + if (cal != null) { + logger.info( + "CalibrationFallbackChain: using ${cal.method} calibration " + + "(confidence ${cal.confidencePercent}%)", + ) + return cal + } else { + logger.info("CalibrationFallbackChain: depth frame returned zero/no-confidence depth at tap point") + } + } else { + logger.info("CalibrationFallbackChain: depth sensor available but no frame ready") + } + }, + ) + } else { + logger.info( + "CalibrationFallbackChain: ARCORE_DEPTH skipped — " + + "isAvailable=${depthSensorProvider.isAvailable}, " + + "tapPoint=$depthTapPoint, imageWidth=$imageWidthPx", + ) + } + + // 4. EXIF focal-length math (±15%) + if (sensorData != null && imageWidthPx > 0.0) { + val cal = ExifCalibrationService.estimate(sensorData, imageWidthPx, depthHintMeters) + if (cal != null) { + logger.info("CalibrationFallbackChain: using EXIF_FOCAL calibration (±15%, confidence 20%)") + return cal + } else { + logger.info("CalibrationFallbackChain: EXIF_FOCAL skipped — focal length data absent in EXIF") + } + } else { + logger.info("CalibrationFallbackChain: EXIF_FOCAL skipped — no sensor data or image width") + } + + // 5. Monocular ML depth (±15%, last resort) + val mlReady = monocularDepthEstimator.isAvailable && mlDepthMap != null + val mlParamsValid = depthTapPoint != null && imageWidthPx > 0.0 && imageHeightPx > 0.0 + if (mlReady && mlParamsValid) { + val cal = CalibrationService.computeFromMLDepth(mlDepthMap, depthTapPoint, imageWidthPx, imageHeightPx) + if (cal != null) { + logger.info( + "CalibrationFallbackChain: using MONOCULAR_ML calibration (±15%, confidence 15%)", + ) + return cal + } else { + logger.info("CalibrationFallbackChain: MONOCULAR_ML depth map returned zero depth at tap point") + } + } else { + logger.info( + "CalibrationFallbackChain: MONOCULAR_ML skipped — " + + "ready=$mlReady, paramsValid=$mlParamsValid", + ) + } + + // 6. No calibration available + logger.info("CalibrationFallbackChain: all methods exhausted — returning NONE") + return Calibration(method = CalibrationMethod.NONE, pixelsPerMeter = 0.0, confidencePercent = 0) + } +} diff --git a/kmp/src/commonMain/kotlin/dev/stapler/stelekit/calibration/CalibrationService.kt b/kmp/src/commonMain/kotlin/dev/stapler/stelekit/calibration/CalibrationService.kt new file mode 100644 index 00000000..10477306 --- /dev/null +++ b/kmp/src/commonMain/kotlin/dev/stapler/stelekit/calibration/CalibrationService.kt @@ -0,0 +1,250 @@ +package dev.stapler.stelekit.calibration + +import dev.stapler.stelekit.model.Calibration +import dev.stapler.stelekit.model.CalibrationMethod +import dev.stapler.stelekit.model.ImageSensorData +import dev.stapler.stelekit.model.NormalizedPoint +import kotlin.math.atan +import kotlin.math.sqrt +import kotlin.math.tan + +/** + * Pure-math calibration computations. + * + * All methods are stateless and platform-agnostic. No platform-specific code here. + * For EXIF-based estimation, use [ExifCalibrationService]. + */ +object CalibrationService { + + /** + * Compute a [Calibration] from a two-point reference line drawn over an object of + * known real-world length. + * + * The pixel start/end are in *normalized* [0,1] image-space coordinates. The + * [imageWidthPx] / [imageHeightPx] values must be the native resolution of the image + * (not the on-screen canvas size). + * + * @param pixelStart normalized start point of the reference line ([0,1] space) + * @param pixelEnd normalized end point of the reference line ([0,1] space) + * @param imageWidthPx image native width in pixels + * @param imageHeightPx image native height in pixels + * @param knownLengthMeters real-world length of the reference object in meters + * @return [Calibration] with [CalibrationMethod.MANUAL_REFERENCE] and + * [Calibration.confidencePercent] = 100, or null if the pixel distance is zero. + */ + fun computeFromReference( + pixelStart: NormalizedPoint, + pixelEnd: NormalizedPoint, + imageWidthPx: Double, + imageHeightPx: Double, + knownLengthMeters: Double, + ): Calibration? { + require(knownLengthMeters > 0.0) { "knownLengthMeters must be positive, got $knownLengthMeters" } + require(imageWidthPx > 0.0) { "imageWidthPx must be positive" } + require(imageHeightPx > 0.0) { "imageHeightPx must be positive" } + + val dx = (pixelEnd.x - pixelStart.x) * imageWidthPx + val dy = (pixelEnd.y - pixelStart.y) * imageHeightPx + val pixelDistance = sqrt(dx * dx + dy * dy) + + if (pixelDistance == 0.0) return null + + val pixelsPerMeter = pixelDistance / knownLengthMeters + return Calibration( + method = CalibrationMethod.MANUAL_REFERENCE, + pixelsPerMeter = pixelsPerMeter, + confidencePercent = 100, + ) + } + + /** + * Compute a [Calibration] by sampling depth from an [DepthFrame] at a normalized tap point. + * + * Used for ARCore depth calibration. The returned [Calibration.confidencePercent] is + * derived from the ARCore per-pixel confidence value (0–255) scaled to [0,100]. + * + * @param depthFrame the ARCore depth frame + * @param tapPointNormalized normalized image coordinates of the user's tap [0,1] + * @param imageWidthPx native image width in pixels (used to compute pixelsPerMeter) + * @return [Calibration] with [CalibrationMethod.ARCORE_DEPTH], or null if depth + * is unavailable at the tap point or confidence is zero. + */ + fun computeFromDepthFrame( + depthFrame: DepthFrame, + tapPointNormalized: NormalizedPoint, + imageWidthPx: Double, + ): Calibration? { + require(imageWidthPx > 0.0) { "imageWidthPx must be positive" } + if (depthFrame.width == 0 || depthFrame.height == 0) return null + + val px = (tapPointNormalized.x * depthFrame.width).toInt().coerceIn(0, depthFrame.width - 1) + val py = (tapPointNormalized.y * depthFrame.height).toInt().coerceIn(0, depthFrame.height - 1) + val idx = py * depthFrame.width + px + + if (idx >= depthFrame.depthMapMm.size) return null + val depthMm = depthFrame.depthMapMm[idx] + if (depthMm <= 0f) return null + + val depthM = depthMm / 1000.0 + + // pixels per meter at this depth using a simple pin-hole model: + // 1 meter at distance D subtends imageWidthPx / (D * 2) pixels if FOV = 90°. + // ARCore depth gives us the actual metric depth so we use it directly as the + // reference scale: pixelsPerMeter = imageWidthPx / depthM (width : 1m at depth D). + // This is intentionally conservative — the badge will show ±8–10 cm confidence. + val pixelsPerMeter = imageWidthPx / depthM + + val confidenceRaw = if (idx < depthFrame.confidenceMap.size) depthFrame.confidenceMap[idx] else 0f + val confidencePercent = ((confidenceRaw / 255f) * 100).toInt().coerceIn(0, 100) + + if (confidencePercent == 0) return null + + return Calibration( + method = CalibrationMethod.ARCORE_DEPTH, + pixelsPerMeter = pixelsPerMeter, + confidencePercent = confidencePercent, + ) + } + + /** + * Compute a [Calibration] from a monocular ML depth map. + * + * The depth map is a flat [FloatArray] in row-major order (same dimensions as the image). + * Values are depth estimates in meters (relative, not absolute — use with caution). + * + * @param depthMap flat FloatArray of depth estimates (meters, relative) + * @param tapPointNormalized normalized image coordinates of the user's tap [0,1] + * @param imageWidthPx native image width in pixels + * @param imageHeightPx native image height in pixels + * @return [Calibration] with [CalibrationMethod.MONOCULAR_ML] and + * [Calibration.confidencePercent] = 15, or null if depth is zero at the tap point. + */ + fun computeFromMLDepth( + depthMap: FloatArray, + tapPointNormalized: NormalizedPoint, + imageWidthPx: Double, + imageHeightPx: Double, + ): Calibration? { + require(imageWidthPx > 0.0) { "imageWidthPx must be positive" } + require(imageHeightPx > 0.0) { "imageHeightPx must be positive" } + + val mapWidth = imageWidthPx.toInt() + val mapHeight = imageHeightPx.toInt() + + if (depthMap.size != mapWidth * mapHeight) return null + + val px = (tapPointNormalized.x * mapWidth).toInt().coerceIn(0, mapWidth - 1) + val py = (tapPointNormalized.y * mapHeight).toInt().coerceIn(0, mapHeight - 1) + val idx = py * mapWidth + px + + val depthM = depthMap[idx].toDouble() + if (depthM <= 0.0) return null + + val pixelsPerMeter = imageWidthPx / depthM + + return Calibration( + method = CalibrationMethod.MONOCULAR_ML, + pixelsPerMeter = pixelsPerMeter, + confidencePercent = 15, + ) + } +} + +/** + * EXIF focal-length based calibration estimation. + * + * This object is separate from [CalibrationService] because it operates on [ImageSensorData] + * which requires no depth frame, and it produces a coarser estimate (±15%). + */ +object ExifCalibrationService { + + /** + * Estimate a [Calibration] from EXIF focal-length data in [sensorData]. + * + * Formula (pinhole camera model): + * ``` + * horizontalFOV = 2 * atan(sensorWidth / (2 * focalLength)) + * pixelsPerMeter at depth D = imageWidth / (2 * D * tan(fovH / 2)) + * ``` + * + * When [depthHintMeters] is null, a standard reference distance of 2.0 m is used. + * + * Returns null if the required EXIF fields are absent. + * + * @param sensorData EXIF data captured at shoot time + * @param depthHintMeters optional depth hint for the pixel-per-meter conversion + * @param imageWidthPx native image width in pixels + */ + fun estimate( + sensorData: ImageSensorData, + imageWidthPx: Double, + depthHintMeters: Double? = null, + ): Calibration? { + // We need focal length. Prefer actual focal length, fall back to 35mm equivalent. + val focalLengthMm = sensorData.focalLengthMm + val focal35mm = sensorData.focalLength35mmEq + + if (focalLengthMm == null && focal35mm == null) return null + if (imageWidthPx <= 0.0) return null + + // Derive horizontal FOV. + // For actual focal length we also need the sensor width (crop-factor math). + // We use the 35mm-equivalent to avoid needing the physical sensor size, since + // full-frame 35mm standard width is 36 mm. + val fovHalfRadians: Double = if (focal35mm != null && focal35mm > 0.0) { + atan(36.0 / (2.0 * focal35mm)) + } else { + // Fall back: use actual focal length assuming a 6.4mm sensor (typical mobile) + val sensorWidthMm = 6.4 + atan(sensorWidthMm / (2.0 * focalLengthMm!!)) + } + + val depthM = depthHintMeters ?: 2.0 + if (depthM <= 0.0) return null + + // pixelsPerMeter = imageWidth / (2 * depth * tan(fovH/2)) + val pixelsPerMeter = imageWidthPx / (2.0 * depthM * tan(fovHalfRadians)) + + if (pixelsPerMeter <= 0.0) return null + + return Calibration( + method = CalibrationMethod.EXIF_FOCAL, + pixelsPerMeter = pixelsPerMeter, + confidencePercent = 20, + ) + } +} + +/** + * A snapshot of depth data from ARCore (Android) or LiDAR (iOS). + * + * All arrays are row-major with dimensions [width] x [height]. + * + * @param depthMapMm depth values in millimeters (positive = in front of camera) + * @param confidenceMap per-pixel confidence [0–255]; 0 = no data, 255 = maximum confidence + * @param width map width in pixels + * @param height map height in pixels + */ +data class DepthFrame( + val depthMapMm: FloatArray, + val confidenceMap: FloatArray, + val width: Int, + val height: Int, +) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is DepthFrame) return false + return width == other.width && + height == other.height && + depthMapMm.contentEquals(other.depthMapMm) && + confidenceMap.contentEquals(other.confidenceMap) + } + + override fun hashCode(): Int { + var result = depthMapMm.contentHashCode() + result = 31 * result + confidenceMap.contentHashCode() + result = 31 * result + width + result = 31 * result + height + return result + } +} diff --git a/kmp/src/commonMain/kotlin/dev/stapler/stelekit/db/ImageImportService.kt b/kmp/src/commonMain/kotlin/dev/stapler/stelekit/db/ImageImportService.kt new file mode 100644 index 00000000..6d140fa6 --- /dev/null +++ b/kmp/src/commonMain/kotlin/dev/stapler/stelekit/db/ImageImportService.kt @@ -0,0 +1,281 @@ +package dev.stapler.stelekit.db + +import arrow.core.Either +import arrow.core.left +import arrow.core.right +import dev.stapler.stelekit.db.sidecar.ImageSidecarManager +import dev.stapler.stelekit.error.DomainError +import dev.stapler.stelekit.model.AnnotationType +import dev.stapler.stelekit.model.Block +import dev.stapler.stelekit.model.ImageAnnotation +import dev.stapler.stelekit.model.ImageSensorData +import dev.stapler.stelekit.model.ImageSource +import dev.stapler.stelekit.model.MeasurementAnnotation +import dev.stapler.stelekit.model.MeasurementUnit +import dev.stapler.stelekit.model.NormalizedPoint +import dev.stapler.stelekit.platform.FileSystem +import dev.stapler.stelekit.platform.sensor.PlatformImageFile +import dev.stapler.stelekit.repository.BlockRepository +import dev.stapler.stelekit.repository.DirectRepositoryWrite +import dev.stapler.stelekit.repository.ImageAnnotationRepository +import dev.stapler.stelekit.repository.JournalService +import dev.stapler.stelekit.repository.MeasurementAnnotationRepository +import dev.stapler.stelekit.util.UuidGenerator +import kotlinx.coroutines.CancellationException +import kotlin.math.roundToInt +import kotlin.time.Clock + +/** + * Orchestrates the full image import pipeline for Epic 2. + * + * Import flow: + * 1. Reserve a stable file path in `/assets/images/` + * 2. Copy image bytes from the temp [PlatformImageFile] path to the reserved path + * 3. Create an [ImageAnnotation] domain object + * 4. Write the JSON sidecar FIRST (per Known Issues transactional write order) + * 5. Save the [ImageAnnotation] to SQLDelight via the repository + * 6. Create an `image_annotation` [Block] on the target page + * 7. Optionally auto-insert the block into today's journal page + * + * Step 4 before step 5 is critical: the sidecar is the authoritative source of truth. + * If the sidecar write fails, the DB row is never written and the error is returned. + */ +class ImageImportService( + private val fileSystem: FileSystem, + private val imageAnnotationRepository: ImageAnnotationRepository? = null, + private val blockRepository: BlockRepository? = null, + private val sidecarManager: ImageSidecarManager? = null, + private val journalService: JournalService? = null, + private val writeActor: DatabaseWriteActor? = null, + private val measurementAnnotationRepository: MeasurementAnnotationRepository? = null, +) { + + /** + * Reserve the target directory for a new image with [uuid] in [graphPath]. + * + * Creates the `assets/images/` directory tree if it does not yet exist. + * + * Returns the resolved file path that the caller should write image bytes to. + */ + fun reservePath(graphPath: String, uuid: String): Either { + val dir = ImageStoragePathResolver.assetsImagesDir(graphPath) + ensureDirectory("$graphPath/assets") + ensureDirectory(dir) + return ImageStoragePathResolver.resolvePath(graphPath, uuid).right() + } + + /** + * Full import pipeline: copy bytes → create annotation → write sidecar → save DB → create block. + * + * @param tempFile Platform-obtained image file ready for import. + * @param graphPath Absolute path (or saf:// URI) of the target graph. + * @param pageUuid UUID of the page that will own the new block. + * @param source How the image was obtained ([ImageSource.CAMERA], [ImageSource.FILE], etc). + * @param insertToJournalPage If `true` AND [journalService] is wired in, also appends the + * block content to today's journal page. + * @return The persisted [ImageAnnotation] on success, or a [DomainError] on failure. + */ + @OptIn(DirectRepositoryWrite::class) + suspend fun import( + tempFile: PlatformImageFile, + graphPath: String, + pageUuid: String, + source: ImageSource = ImageSource.FILE, + insertToJournalPage: Boolean = false, + ): Either { + if (imageAnnotationRepository == null) { + return DomainError.DatabaseError.WriteFailed( + "ImageImportService: imageAnnotationRepository not wired" + ).left() + } + if (sidecarManager == null) { + return DomainError.FileSystemError.WriteFailed( + "", + "ImageImportService: imageSidecarManager not wired" + ).left() + } + + val annotationUuid = UuidGenerator.generateV7() + val blockUuid = UuidGenerator.generateV7() + val now = Clock.System.now() + + // Step 1: Reserve stable path and ensure directory exists + val destPath = reservePath(graphPath, annotationUuid) + .fold({ return it.left() }, { it }) + + // Step 2: Copy bytes from temp location to graph assets + copyImageBytes(tempFile.path, destPath).fold( + ifLeft = { return it.left() }, + ifRight = { /* success — continue */ }, + ) + + // Step 3: Build the domain object. + // Merge EXIF data from PlatformImageFile with motion sensor data (GPS, bearing, + // pitch/roll) captured at the moment of image capture (Story 8.1.5). + val capturedSensorData = tempFile.sensorData + val annotation = ImageAnnotation( + uuid = annotationUuid, + blockUuid = blockUuid, + pageUuid = pageUuid, + graphPath = graphPath, + filePath = destPath, + source = source, + capturedAtMs = tempFile.capturedAtMs, + importedAtMs = now.toEpochMilliseconds(), + unit = MeasurementUnit.METERS, + sensorData = ImageSensorData( + // GPS + motion sensor data from MotionSensorProvider snapshot at capture time + latLng = capturedSensorData?.latLng, + altitudeM = capturedSensorData?.altitudeM, + bearingDeg = capturedSensorData?.bearingDeg, + pitchDeg = capturedSensorData?.pitchDeg, + rollDeg = capturedSensorData?.rollDeg, + // EXIF data from the image file + focalLengthMm = tempFile.focalLengthMm ?: capturedSensorData?.focalLengthMm, + focalLength35mmEq = tempFile.focalLength35mmEq ?: capturedSensorData?.focalLength35mmEq, + cameraMake = tempFile.cameraMake ?: capturedSensorData?.cameraMake, + cameraModel = tempFile.cameraModel ?: capturedSensorData?.cameraModel, + ), + ) + + // Step 4: Write sidecar BEFORE DB insert (Known Issues transactional write order) + sidecarManager.writeSidecar(annotation, emptyList()).fold( + ifLeft = { err -> + // Sidecar failed — roll back the copied file and return error + fileSystem.deleteFile(destPath) + return err.left() + }, + ifRight = { /* continue */ }, + ) + + // Step 5: Save annotation to DB + imageAnnotationRepository.saveImageAnnotation(annotation).fold( + ifLeft = { err -> + // DB write failed; sidecar already written. Leave sidecar in place — + // ImageSidecarIndexer can recover the DB row from the sidecar on next startup. + return err.left() + }, + ifRight = { /* continue */ }, + ) + + // Step 6: Create the image_annotation block + val filename = destPath.substringAfterLast("/") + val relPath = "../assets/images/$filename" + val blockContent = "![]($relPath)" + val blockProperties = mapOf( + "image-id" to annotationUuid, + "calibration" to "none", + "unit" to annotation.unit.name.lowercase(), + ) + val block = Block( + uuid = blockUuid, + pageUuid = pageUuid, + content = blockContent, + position = 0, + createdAt = now, + updatedAt = now, + properties = blockProperties, + blockType = "image_annotation", + ) + + saveBlock(block).fold( + ifLeft = { err -> return err.left() }, + ifRight = { /* continue */ }, + ) + + // Step 6b: Auto-create compass bearing annotation (Story 8.3) + // If bearing data is present in sensor data, create an initial LABEL annotation + // positioned at the top-right corner of the image (~0.85, 0.05 normalized coords). + val bearingDeg = annotation.sensorData.bearingDeg + if (bearingDeg != null) { + val bearingInt = bearingDeg.roundToInt() + val bearingLabel = "Bearing: ${bearingInt}°N" + val bearingAnnotation = MeasurementAnnotation( + uuid = UuidGenerator.generateV7(), + imageUuid = annotationUuid, + annotationType = AnnotationType.LABEL, + normalizedPoints = listOf(NormalizedPoint(0.85, 0.05)), + label = bearingLabel, + ) + try { + measurementAnnotationRepository?.saveMeasurementAnnotation(bearingAnnotation) + } catch (_: CancellationException) { + throw CancellationException() + } catch (_: Exception) { + // Non-fatal: bearing annotation failure must not fail the import + } + } + + // Step 7: Auto-insert into today's journal page (camera import, Story 2.3.3) + if (insertToJournalPage && source == ImageSource.CAMERA) { + try { + journalService?.appendToToday(blockContent) + } catch (e: CancellationException) { + throw e + } catch (_: Exception) { + // Non-fatal: journal append failure must not fail the import + } + } + + return annotation.right() + } + + // ── Helpers ────────────────────────────────────────────────────────────── + + /** + * Copy raw bytes from [srcPath] to [destPath]. + * + * Uses [FileSystem.readFileBytes] and [FileSystem.writeFileBytes] so the bytes + * never pass through a String decode/encode cycle. + */ + private fun copyImageBytes(srcPath: String, destPath: String): Either { + return try { + val bytes = fileSystem.readFileBytes(srcPath) + ?: return DomainError.FileSystemError.NotFound(srcPath).left() + val written = fileSystem.writeFileBytes(destPath, bytes) + if (written) Unit.right() + else DomainError.FileSystemError.WriteFailed(destPath, "writeFileBytes returned false").left() + } catch (e: CancellationException) { + throw e + } catch (e: UnsupportedOperationException) { + // Platform hasn't implemented readFileBytes/writeFileBytes — attempt text copy as fallback + copyBytesViaText(srcPath, destPath) + } catch (e: Exception) { + DomainError.FileSystemError.WriteFailed(destPath, e.message ?: "unknown").left() + } + } + + /** Text-path fallback for platforms without byte IO. */ + private fun copyBytesViaText(srcPath: String, destPath: String): Either { + return try { + val content = fileSystem.readFile(srcPath) + ?: return DomainError.FileSystemError.NotFound(srcPath).left() + val written = fileSystem.writeFile(destPath, content) + if (written) Unit.right() + else DomainError.FileSystemError.WriteFailed(destPath, "writeFile returned false").left() + } catch (e: CancellationException) { + throw e + } catch (e: Exception) { + DomainError.FileSystemError.WriteFailed(destPath, e.message ?: "unknown").left() + } + } + + @OptIn(DirectRepositoryWrite::class) + private suspend fun saveBlock(block: Block): Either { + val actor = writeActor + val repo = blockRepository + return when { + actor != null -> actor.saveBlock(block) + repo != null -> repo.saveBlock(block) + else -> DomainError.DatabaseError.WriteFailed( + "BlockRepository not wired — cannot create image_annotation block" + ).left() + } + } + + private fun ensureDirectory(path: String) { + if (!fileSystem.directoryExists(path)) { + fileSystem.createDirectory(path) + } + } +} diff --git a/kmp/src/commonMain/kotlin/dev/stapler/stelekit/db/ImageStoragePathResolver.kt b/kmp/src/commonMain/kotlin/dev/stapler/stelekit/db/ImageStoragePathResolver.kt new file mode 100644 index 00000000..f701e939 --- /dev/null +++ b/kmp/src/commonMain/kotlin/dev/stapler/stelekit/db/ImageStoragePathResolver.kt @@ -0,0 +1,36 @@ +package dev.stapler.stelekit.db + +import kotlin.time.Clock +import kotlinx.datetime.TimeZone +import kotlinx.datetime.toLocalDateTime + +/** + * Resolves the on-disk storage path for an image file inside a graph. + * + * Path convention: `/assets/images/-.jpg` + * + * - Date is the calendar date at the moment of resolution (local time zone). + * - UUID prefix is the first 8 characters of [uuid] (enough to be human-readable; + * collisions within a single day are astronomically unlikely). + * - The `.jpg` extension is used even if the source is a PNG — images are always + * re-encoded to JPEG before storage to control file size. + */ +object ImageStoragePathResolver { + + /** + * Compute the full on-disk path for an image with the given [uuid] stored in [graphPath]. + * + * Example: `/home/user/my-graph/assets/images/2026-05-16-a3f8b2c1.jpg` + */ + fun resolvePath(graphPath: String, uuid: String): String { + val now = Clock.System.now() + val date = now.toLocalDateTime(TimeZone.currentSystemDefault()).date + val dateStr = "${date.year}-${date.monthNumber.toString().padStart(2, '0')}-${date.dayOfMonth.toString().padStart(2, '0')}" + val uuidPrefix = uuid.replace("-", "").take(8) + return "$graphPath/assets/images/$dateStr-$uuidPrefix.jpg" + } + + /** Directory under which all image assets are stored. */ + fun assetsImagesDir(graphPath: String): String = + "$graphPath/assets/images" +} diff --git a/kmp/src/commonMain/kotlin/dev/stapler/stelekit/db/RestrictedDatabaseQueries.kt b/kmp/src/commonMain/kotlin/dev/stapler/stelekit/db/RestrictedDatabaseQueries.kt index 88945a86..b493b4cf 100644 --- a/kmp/src/commonMain/kotlin/dev/stapler/stelekit/db/RestrictedDatabaseQueries.kt +++ b/kmp/src/commonMain/kotlin/dev/stapler/stelekit/db/RestrictedDatabaseQueries.kt @@ -427,4 +427,105 @@ class RestrictedDatabaseQueries(private val queries: SteleDatabaseQueries) { @DirectSqlWrite suspend fun deleteGitConfig(graph_id: String) = queries.deleteGitConfig(graph_id) + + // ── Image annotation writes ─────────────────────────────────────────────── + + @DirectSqlWrite + suspend fun insertImageAnnotation( + uuid: String, + block_uuid: String, + page_uuid: String, + graph_path: String, + file_path: String, + thumbnail_path: String?, + source: String, + source_uri: String?, + captured_at_ms: Long?, + imported_at_ms: Long, + calibration_method: String, + pixels_per_meter: Double, + calibration_confidence_pct: Long, + unit: String, + tags: String, + lat_lng: String?, + altitude_m: Double?, + bearing_deg: Double?, + pitch_deg: Double?, + roll_deg: Double?, + focal_length_mm: Double?, + focal_length_35mm_eq: Double?, + camera_make: String?, + camera_model: String?, + ): Long = queries.insertImageAnnotation( + uuid, block_uuid, page_uuid, graph_path, file_path, thumbnail_path, + source, source_uri, captured_at_ms, imported_at_ms, + calibration_method, pixels_per_meter, calibration_confidence_pct, + unit, tags, lat_lng, altitude_m, bearing_deg, pitch_deg, roll_deg, + focal_length_mm, focal_length_35mm_eq, camera_make, camera_model, + ) + + @DirectSqlWrite + suspend fun updateImageAnnotation( + block_uuid: String, + page_uuid: String, + graph_path: String, + file_path: String, + thumbnail_path: String?, + source: String, + source_uri: String?, + captured_at_ms: Long?, + imported_at_ms: Long, + calibration_method: String, + pixels_per_meter: Double, + calibration_confidence_pct: Long, + unit: String, + tags: String, + lat_lng: String?, + altitude_m: Double?, + bearing_deg: Double?, + pitch_deg: Double?, + roll_deg: Double?, + focal_length_mm: Double?, + focal_length_35mm_eq: Double?, + camera_make: String?, + camera_model: String?, + uuid: String, + ): Long = queries.updateImageAnnotation( + block_uuid, page_uuid, graph_path, file_path, thumbnail_path, + source, source_uri, captured_at_ms, imported_at_ms, + calibration_method, pixels_per_meter, calibration_confidence_pct, + unit, tags, lat_lng, altitude_m, bearing_deg, pitch_deg, roll_deg, + focal_length_mm, focal_length_35mm_eq, camera_make, camera_model, + uuid, + ) + + @DirectSqlWrite + suspend fun deleteImageAnnotation(uuid: String): Long = + queries.deleteImageAnnotation(uuid) + + // ── Measurement annotation writes ───────────────────────────────────────── + + @DirectSqlWrite + suspend fun insertMeasurementAnnotation( + uuid: String, + image_uuid: String, + annotation_type: String, + normalized_points: String, + value_meters: Double?, + value_display: String?, + label: String?, + color_hex: String, + ble_device_id: String?, + ): Long = queries.insertMeasurementAnnotation( + uuid, image_uuid, annotation_type, normalized_points, + value_meters, value_display, label, color_hex, ble_device_id, + ) + + @DirectSqlWrite + suspend fun deleteMeasurementsForImage(image_uuid: String): Long = + queries.deleteMeasurementsForImage(image_uuid) + + @DirectSqlWrite + suspend fun deleteMeasurementAnnotation(uuid: String): Long = + queries.deleteMeasurementAnnotation(uuid) } diff --git a/kmp/src/commonMain/kotlin/dev/stapler/stelekit/db/sidecar/ImageSidecarIndexer.kt b/kmp/src/commonMain/kotlin/dev/stapler/stelekit/db/sidecar/ImageSidecarIndexer.kt new file mode 100644 index 00000000..b3593215 --- /dev/null +++ b/kmp/src/commonMain/kotlin/dev/stapler/stelekit/db/sidecar/ImageSidecarIndexer.kt @@ -0,0 +1,87 @@ +package dev.stapler.stelekit.db.sidecar + +import arrow.core.Either +import arrow.core.left +import arrow.core.right +import dev.stapler.stelekit.error.DomainError +import dev.stapler.stelekit.logging.Logger +import dev.stapler.stelekit.platform.FileSystem +import dev.stapler.stelekit.repository.DirectRepositoryWrite +import dev.stapler.stelekit.repository.ImageAnnotationRepository +import dev.stapler.stelekit.repository.MeasurementAnnotationRepository +import kotlinx.coroutines.CancellationException +import kotlinx.serialization.json.Json + +/** + * Rebuilds the SQLDelight [image_annotations] and [measurement_annotations] tables by + * walking all `*.measure.json` sidecar files in `/.stelekit/images/`. + * + * This is the **recovery path** used when the SQLite database is lost (e.g. git clean, + * accidental deletion, or DB corruption). The sidecar files are the authoritative source. + * + * Call [rebuildFromSidecars] at graph-open time if the DB is empty but sidecar files exist. + */ +class ImageSidecarIndexer( + private val fileSystem: FileSystem, + private val imageAnnotationRepository: ImageAnnotationRepository, + private val measurementAnnotationRepository: MeasurementAnnotationRepository, +) { + private val logger = Logger("ImageSidecarIndexer") + private val json = Json { ignoreUnknownKeys = true } + + /** + * Scan all `.measure.json` files under `/.stelekit/images/` and upsert + * each into the repository. + * + * Returns the number of sidecars successfully upserted, or [DomainError] on a + * hard failure (individual parse errors are logged and skipped). + */ + @OptIn(DirectRepositoryWrite::class) + suspend fun rebuildFromSidecars(graphPath: String): Either { + val imagesDir = "$graphPath/.stelekit/images" + if (!fileSystem.directoryExists(imagesDir)) { + logger.info("ImageSidecarIndexer: no sidecar directory at $imagesDir, nothing to rebuild") + return 0.right() + } + + val files = try { + fileSystem.listFiles(imagesDir) + } catch (e: CancellationException) { + throw e + } catch (e: Exception) { + return DomainError.FileSystemError.ReadFailed(imagesDir, e.message ?: "unknown").left() + } + + var upserted = 0 + for (fileName in files) { + if (!fileName.endsWith(".measure.json")) continue + val fullPath = "$imagesDir/$fileName" + try { + val content = fileSystem.readFile(fullPath) ?: continue + val sidecar = json.decodeFromString(SidecarFile.serializer(), content) + val annotation = sidecar.toDomainAnnotation() + val measurements = sidecar.toDomainMeasurements() + + // Upsert: delete first (no-op if absent), then insert + imageAnnotationRepository.deleteImageAnnotation(annotation.uuid) + val saveResult = imageAnnotationRepository.saveImageAnnotation(annotation) + if (saveResult.isLeft()) { + saveResult.onLeft { err -> + logger.warn("ImageSidecarIndexer: failed to upsert annotation ${annotation.uuid}: ${err.message}") + } + continue + } + measurementAnnotationRepository.deleteMeasurementsForImage(annotation.uuid) + measurementAnnotationRepository.saveMeasurements(annotation.uuid, measurements) + upserted++ + } catch (e: CancellationException) { + throw e + } catch (e: Exception) { + logger.warn("ImageSidecarIndexer: skipping malformed sidecar $fullPath: ${e.message}") + } + } + + logger.info("ImageSidecarIndexer: rebuilt $upserted image annotations from sidecars") + return upserted.right() + } +} diff --git a/kmp/src/commonMain/kotlin/dev/stapler/stelekit/db/sidecar/ImageSidecarManager.kt b/kmp/src/commonMain/kotlin/dev/stapler/stelekit/db/sidecar/ImageSidecarManager.kt new file mode 100644 index 00000000..ed62af23 --- /dev/null +++ b/kmp/src/commonMain/kotlin/dev/stapler/stelekit/db/sidecar/ImageSidecarManager.kt @@ -0,0 +1,204 @@ +package dev.stapler.stelekit.db.sidecar + +import arrow.core.Either +import arrow.core.left +import arrow.core.right +import dev.stapler.stelekit.error.DomainError +import dev.stapler.stelekit.model.AnnotationType +import dev.stapler.stelekit.model.Calibration +import dev.stapler.stelekit.model.CalibrationMethod +import dev.stapler.stelekit.model.ImageAnnotation +import dev.stapler.stelekit.model.ImageSensorData +import dev.stapler.stelekit.model.ImageSource +import dev.stapler.stelekit.model.MeasurementAnnotation +import dev.stapler.stelekit.model.MeasurementUnit +import dev.stapler.stelekit.model.NormalizedPoint +import dev.stapler.stelekit.platform.FileSystem +import kotlinx.coroutines.CancellationException +import kotlinx.serialization.json.Json + +/** + * Reads and writes `.measure.json` sidecar files for image annotations. + * + * Sidecar path convention: `/.stelekit/images/.measure.json` + * + * IMPORTANT: always call [writeSidecar] BEFORE the SQLDelight row insert. + * If the sidecar write fails, return the error without touching SQLDelight. + * This ordering guarantees the sidecar is the authoritative record — + * [ImageSidecarIndexer.rebuildFromSidecars] can always reconstruct the DB. + */ +class ImageSidecarManager( + private val fileSystem: FileSystem, +) { + private val json = Json { ignoreUnknownKeys = true; prettyPrint = false } + + /** + * Serialize [annotation] + [measurements] to JSON and write to the sidecar path. + * + * Returns [Either.Left] if the file system write fails. + */ + fun writeSidecar( + annotation: ImageAnnotation, + measurements: List, + ): Either { + return try { + val path = sidecarPath(annotation.graphPath, annotation.uuid) + ensureSidecarDir(annotation.graphPath) + + val sidecar = SidecarFile( + schemaVersion = 1, + imageAnnotation = annotation.toSidecar(), + measurements = measurements.map { it.toSidecar() }, + ) + val jsonString = json.encodeToString(SidecarFile.serializer(), sidecar) + val written = fileSystem.writeFile(path, jsonString) + if (written) Unit.right() + else DomainError.FileSystemError.WriteFailed(path, "FileSystem.writeFile returned false").left() + } catch (e: CancellationException) { + throw e + } catch (e: Exception) { + DomainError.FileSystemError.WriteFailed( + sidecarPath(annotation.graphPath, annotation.uuid), + e.message ?: "unknown" + ).left() + } + } + + /** + * Read and deserialize the sidecar for [uuid] in [graphPath]. + * + * Returns `null` wrapped in [Either.Right] when the file does not exist (not an error). + * Returns [Either.Left] only for malformed JSON or I/O errors. + */ + fun readSidecar(graphPath: String, uuid: String): Either { + val path = sidecarPath(graphPath, uuid) + return try { + val content = fileSystem.readFile(path) ?: return null.right() + val sidecar = json.decodeFromString(SidecarFile.serializer(), content) + sidecar.right() + } catch (e: CancellationException) { + throw e + } catch (e: kotlinx.serialization.SerializationException) { + DomainError.ParseError.InvalidSyntax("Malformed sidecar JSON at $path: ${e.message}").left() + } catch (e: Exception) { + DomainError.FileSystemError.ReadFailed(path, e.message ?: "unknown").left() + } + } + + /** Compute the sidecar file path for the given graph and annotation UUID. */ + fun sidecarPath(graphPath: String, uuid: String): String = + "$graphPath/.stelekit/images/$uuid.measure.json" + + private fun ensureSidecarDir(graphPath: String) { + val steleDir = "$graphPath/.stelekit" + val imagesDir = "$steleDir/images" + if (!fileSystem.directoryExists(steleDir)) fileSystem.createDirectory(steleDir) + if (!fileSystem.directoryExists(imagesDir)) fileSystem.createDirectory(imagesDir) + } +} + +// ── Domain → sidecar conversions ────────────────────────────────────────────── + +private fun ImageAnnotation.toSidecar(): SidecarImageAnnotation { + val latLngStr = sensorData.latLng?.let { (lat, lng) -> "$lat,$lng" } + return SidecarImageAnnotation( + uuid = uuid, + blockUuid = blockUuid, + pageUuid = pageUuid, + graphPath = graphPath, + filePath = filePath, + thumbnailPath = thumbnailPath, + source = source.name, + sourceUri = sourceUri, + capturedAtMs = capturedAtMs, + importedAtMs = importedAtMs, + calibrationMethod = calibration.method.name, + pixelsPerMeter = calibration.pixelsPerMeter, + calibrationConfidencePct = calibration.confidencePercent, + unit = unit.name, + tags = tags, + latLng = latLngStr, + altitudeM = sensorData.altitudeM, + bearingDeg = sensorData.bearingDeg, + pitchDeg = sensorData.pitchDeg, + rollDeg = sensorData.rollDeg, + focalLengthMm = sensorData.focalLengthMm, + focalLength35mmEq = sensorData.focalLength35mmEq, + cameraMake = sensorData.cameraMake, + cameraModel = sensorData.cameraModel, + ) +} + +private fun MeasurementAnnotation.toSidecar(): SidecarMeasurement = + SidecarMeasurement( + uuid = uuid, + imageUuid = imageUuid, + annotationType = annotationType.name, + normalizedPoints = normalizedPoints.map { SidecarPoint(it.x, it.y) }, + valueMeters = valueMeters, + valueDisplay = valueDisplay, + label = label, + colorHex = colorHex, + bleDeviceId = bleDeviceId, + ) + +// ── Sidecar → domain conversions ────────────────────────────────────────────── + +fun SidecarFile.toDomainAnnotation(): ImageAnnotation { + val ann = imageAnnotation + val latLngPair = ann.latLng?.let { raw -> + val parts = raw.split(",") + if (parts.size == 2) { + val lat = parts[0].trim().toDoubleOrNull() + val lng = parts[1].trim().toDoubleOrNull() + if (lat != null && lng != null) lat to lng else null + } else null + } + return ImageAnnotation( + uuid = ann.uuid, + blockUuid = ann.blockUuid, + pageUuid = ann.pageUuid, + graphPath = ann.graphPath, + filePath = ann.filePath, + thumbnailPath = ann.thumbnailPath, + source = runCatching { ImageSource.valueOf(ann.source) }.getOrDefault(ImageSource.FILE), + sourceUri = ann.sourceUri, + capturedAtMs = ann.capturedAtMs, + importedAtMs = ann.importedAtMs, + calibration = Calibration( + method = runCatching { CalibrationMethod.valueOf(ann.calibrationMethod) }.getOrDefault(CalibrationMethod.NONE), + pixelsPerMeter = ann.pixelsPerMeter, + confidencePercent = ann.calibrationConfidencePct, + ), + unit = runCatching { MeasurementUnit.valueOf(ann.unit) }.getOrDefault(MeasurementUnit.METERS), + tags = ann.tags, + sensorData = ImageSensorData( + latLng = latLngPair, + altitudeM = ann.altitudeM, + bearingDeg = ann.bearingDeg, + pitchDeg = ann.pitchDeg, + rollDeg = ann.rollDeg, + focalLengthMm = ann.focalLengthMm, + focalLength35mmEq = ann.focalLength35mmEq, + cameraMake = ann.cameraMake, + cameraModel = ann.cameraModel, + ), + ) +} + +fun SidecarFile.toDomainMeasurements(): List = + measurements.map { m -> + MeasurementAnnotation( + uuid = m.uuid, + imageUuid = m.imageUuid, + annotationType = runCatching { AnnotationType.valueOf(m.annotationType) }.getOrDefault(AnnotationType.DISTANCE), + normalizedPoints = m.normalizedPoints + .filter { it.x in 0.0..1.0 && it.y in 0.0..1.0 } + .map { NormalizedPoint(it.x, it.y) }, + valueMeters = m.valueMeters, + valueDisplay = m.valueDisplay, + label = m.label, + colorHex = m.colorHex, + bleDeviceId = m.bleDeviceId, + ) + } diff --git a/kmp/src/commonMain/kotlin/dev/stapler/stelekit/db/sidecar/ImageSidecarSchema.kt b/kmp/src/commonMain/kotlin/dev/stapler/stelekit/db/sidecar/ImageSidecarSchema.kt new file mode 100644 index 00000000..0243e6c9 --- /dev/null +++ b/kmp/src/commonMain/kotlin/dev/stapler/stelekit/db/sidecar/ImageSidecarSchema.kt @@ -0,0 +1,70 @@ +package dev.stapler.stelekit.db.sidecar + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable + +/** + * JSON sidecar schema v1 for image annotations. + * + * The sidecar file at `/.stelekit/images/.measure.json` is the + * authoritative portable source of truth for measurement data. The SQLDelight row is + * a query-optimised cache that can always be rebuilt from the sidecar via + * [ImageSidecarIndexer.rebuildFromSidecars]. + * + * IMPORTANT: write the sidecar BEFORE committing the SQLDelight row so that a crash + * between the two writes never leaves a DB row without a corresponding sidecar file. + */ + +@Serializable +data class SidecarFile( + @SerialName("schemaVersion") val schemaVersion: Int = 1, + @SerialName("imageAnnotation") val imageAnnotation: SidecarImageAnnotation, + @SerialName("measurements") val measurements: List = emptyList(), +) + +@Serializable +data class SidecarImageAnnotation( + @SerialName("uuid") val uuid: String, + @SerialName("blockUuid") val blockUuid: String, + @SerialName("pageUuid") val pageUuid: String, + @SerialName("graphPath") val graphPath: String, + @SerialName("filePath") val filePath: String, + @SerialName("thumbnailPath") val thumbnailPath: String? = null, + @SerialName("source") val source: String = "FILE", + @SerialName("sourceUri") val sourceUri: String? = null, + @SerialName("capturedAtMs") val capturedAtMs: Long? = null, + @SerialName("importedAtMs") val importedAtMs: Long = 0L, + @SerialName("calibrationMethod") val calibrationMethod: String = "NONE", + @SerialName("pixelsPerMeter") val pixelsPerMeter: Double = 0.0, + @SerialName("calibrationConfidencePct") val calibrationConfidencePct: Int = 0, + @SerialName("unit") val unit: String = "METERS", + @SerialName("tags") val tags: List = emptyList(), + @SerialName("latLng") val latLng: String? = null, + @SerialName("altitudeM") val altitudeM: Double? = null, + @SerialName("bearingDeg") val bearingDeg: Double? = null, + @SerialName("pitchDeg") val pitchDeg: Double? = null, + @SerialName("rollDeg") val rollDeg: Double? = null, + @SerialName("focalLengthMm") val focalLengthMm: Double? = null, + @SerialName("focalLength35mmEq") val focalLength35mmEq: Double? = null, + @SerialName("cameraMake") val cameraMake: String? = null, + @SerialName("cameraModel") val cameraModel: String? = null, +) + +@Serializable +data class SidecarMeasurement( + @SerialName("uuid") val uuid: String, + @SerialName("imageUuid") val imageUuid: String, + @SerialName("annotationType") val annotationType: String, + @SerialName("normalizedPoints") val normalizedPoints: List = emptyList(), + @SerialName("valueMeters") val valueMeters: Double? = null, + @SerialName("valueDisplay") val valueDisplay: String? = null, + @SerialName("label") val label: String? = null, + @SerialName("colorHex") val colorHex: String = "#FF0000", + @SerialName("bleDeviceId") val bleDeviceId: String? = null, +) + +@Serializable +data class SidecarPoint( + @SerialName("x") val x: Double, + @SerialName("y") val y: Double, +) diff --git a/kmp/src/commonMain/kotlin/dev/stapler/stelekit/domain/MeasurementPropertySyncer.kt b/kmp/src/commonMain/kotlin/dev/stapler/stelekit/domain/MeasurementPropertySyncer.kt new file mode 100644 index 00000000..e79bb176 --- /dev/null +++ b/kmp/src/commonMain/kotlin/dev/stapler/stelekit/domain/MeasurementPropertySyncer.kt @@ -0,0 +1,139 @@ +// Copyright (c) 2026 Tyler Stapler +// SPDX-License-Identifier: Elastic-2.0 + +package dev.stapler.stelekit.domain + +import dev.stapler.stelekit.model.AnnotationType +import dev.stapler.stelekit.model.CalibrationMethod +import dev.stapler.stelekit.model.ImageAnnotation +import dev.stapler.stelekit.model.MeasurementAnnotation +import dev.stapler.stelekit.model.MeasurementUnit + +// Regex compiled once at file level — prevents RegexInLambda detekt warning. +private val LABEL_SANITIZE_REGEX = Regex("[^a-zA-Z0-9-_]") + +/** + * Derives a [Map] of Logseq-compatible block properties from an [ImageAnnotation] and its + * list of [MeasurementAnnotation]s. + * + * Callers should merge the returned map into the parent block's existing properties and persist + * it via [BlockRepository.saveBlock] or [DatabaseWriteActor.saveBlock]. + * + * Property conventions: + * - Named annotations (non-blank [MeasurementAnnotation.label]) per annotation type: + * - DISTANCE → `measure-