ADFA-1552 experimental computer vision for layout xml

app/build.gradle.kts

@@ -287,6 +287,7 @@ dependencies {
 
 	implementation(projects.layouteditor)
 	implementation(projects.idetooltips)
+ implementation(projects.cvImageToXml)
 
 	// This is to build the tooling-api-impl project before the app is built
 	// So we always copy the latest JAR file to assets

app/src/main/java/com/itsaky/androidide/actions/etc/GenerateXMLAction.kt

@@ -0,0 +1,113 @@
+package com.itsaky.androidide.actions.etc
+
+import android.content.Context
+import android.content.Intent
+import android.view.MenuItem
+import androidx.core.content.ContextCompat
+import org.appdevforall.codeonthego.computervision.ui.ComputerVisionActivity
+import com.android.aaptcompiler.AaptResourceType.LAYOUT
+import com.android.aaptcompiler.extractPathData
+import com.blankj.utilcode.util.KeyboardUtils
+import com.itsaky.androidide.actions.ActionData
+import com.itsaky.androidide.actions.EditorRelatedAction
+import com.itsaky.androidide.actions.markInvisible
+import com.itsaky.androidide.activities.editor.EditorHandlerActivity
+import com.itsaky.androidide.editor.ui.IDEEditor
+import com.itsaky.androidide.resources.R
+import io.sentry.Sentry
+
+
+import java.io.File
+
+class GenerateXMLAction(context: Context, override val order: Int) : EditorRelatedAction() {
+
+ override val id: String = ID
+ override fun retrieveTooltipTag(isReadOnlyContext: Boolean): String = ""
+ override var requiresUIThread: Boolean = false
+
+ companion object {
+ const val ID = "ide.editor.generatexml"
+ const val EXTRA_LAYOUT_FILE_PATH = "com.example.images.LAYOUT_FILE_PATH"
+ const val EXTRA_LAYOUT_FILE_NAME = "com.example.images.LAYOUT_FILE_NAME"
+ }
+
+ init {
+ label = context.getString(R.string.title_image_to_layout)
+ icon = ContextCompat.getDrawable(context, R.drawable.ic_computer_vision)
+ }
+
+ override fun prepare(data: ActionData) {
+ super.prepare(data)
+
+ runCatching {
+ val viewModel = data.requireActivity().editorViewModel
+ if (viewModel.isInitializing) {
+ visible = true
+ enabled = false
+ return
+ }
+
+ if (!visible) {
+ return
+ }
+
+ val editor = data.requireEditor()
+ val file = editor.file
+ file?.let {
+ val isXml = file.name.endsWith(".xml")
+
+ if (!isXml) {
+ markInvisible()
+ return
+ }
+
+ val type = try {
+ extractPathData(file).type
+ } catch (err: Throwable) {
+ markInvisible()
+ return
+ }
+ visible = type == LAYOUT
+ enabled = visible
+ }
+ }.onFailure {
+ Sentry.captureException(it)
+ }
+
+ }
+
+ override suspend fun execAction(data: ActionData): Any {
+ return true
+ }
+
+ override fun getShowAsActionFlags(data: ActionData): Int {
+ val activity = data.getActivity() ?: return super.getShowAsActionFlags(data)
+ return if (KeyboardUtils.isSoftInputVisible(activity)) {
+ MenuItem.SHOW_AS_ACTION_IF_ROOM
+ } else {
+ MenuItem.SHOW_AS_ACTION_ALWAYS
+ }
+ }
+
+
+ override fun postExec(data: ActionData, result: Any) {
+ val activity = data.requireActivity()
+ activity.let {
+ activity.navigateComputerVisionActivity(data.requireEditor().file!!)
+ }
+ }
+
+ private fun EditorHandlerActivity.navigateComputerVisionActivity(file: File) {
+ val intent = Intent(this, ComputerVisionActivity::class.java).apply {
+ putExtra(EXTRA_LAYOUT_FILE_PATH, file.absolutePath)
+ putExtra(EXTRA_LAYOUT_FILE_NAME, file.name)
+ }
+ uiDesignerResultLauncher?.launch(intent)
+ }
+
+ private fun ActionData.requireEditor(): IDEEditor {
+ return this.getEditor() ?: throw IllegalArgumentException(
+ "An editor instance is required but none was provided"
+ )
+ }
+}

app/src/main/java/com/itsaky/androidide/app/DeviceProtectedApplicationLoader.kt

@@ -3,6 +3,7 @@ package com.itsaky.androidide.app
 import androidx.lifecycle.DefaultLifecycleObserver
 import androidx.lifecycle.LifecycleOwner
 import androidx.lifecycle.ProcessLifecycleOwner
+import org.appdevforall.codeonthego.computervision.di.computerVisionModule
 import com.itsaky.androidide.BuildConfig
 import com.itsaky.androidide.analytics.IAnalyticsManager
 import com.itsaky.androidide.app.strictmode.StrictModeConfig
@@ -67,7 +68,7 @@ internal object DeviceProtectedApplicationLoader :
 
  startKoin {
  androidContext(app)
- modules(coreModule, pluginModule)
+ modules(coreModule, pluginModule, computerVisionModule)
  }
 
  SentryAndroid.init(app) { options ->

app/src/main/java/com/itsaky/androidide/utils/EditorActivityActions.kt

@@ -55,6 +55,7 @@ import com.itsaky.androidide.actions.filetree.RenameAction
 import com.itsaky.androidide.actions.text.RedoAction
 import com.itsaky.androidide.actions.text.UndoAction
 import com.itsaky.androidide.actions.PluginActionItem
+import com.itsaky.androidide.actions.etc.GenerateXMLAction
 import com.itsaky.androidide.plugins.extensions.UIExtension
 import com.itsaky.androidide.plugins.manager.core.PluginManager
 
@@ -95,6 +96,7 @@ class EditorActivityActions {
  registry.registerAction(FindInProjectAction(context, order++))
  registry.registerAction(LaunchAppAction(context, order++))
  registry.registerAction(DisconnectLogSendersAction(context, order++))
+ registry.registerAction(action= GenerateXMLAction(context, order=order++))
 
  // Plugin contributions
  order = registerPluginActions(context, registry, order)

cv-image-to-xml/build.gradle.kts

@@ -0,0 +1,51 @@
+plugins {
+ id("com.android.library")
+ id("org.jetbrains.kotlin.android")
+}
+
+android {
+ namespace = "org.appdevforall.codeonthego.computervision"
+
+ defaultConfig {
+ testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
+ }
+
+ buildTypes {
+ release {
+ isMinifyEnabled = false
+ proguardFiles(
+ getDefaultProguardFile("proguard-android-optimize.txt"),
+ "proguard-rules.pro"
+ )
+ }
+ }
+
+ buildFeatures {
+ viewBinding = true
+ }
+}
+
+dependencies {
+ implementation(libs.androidx.core.ktx)
+ implementation(libs.androidx.appcompat)
+ implementation(libs.google.material)
+ implementation(libs.androidx.activity.ktx)
+ implementation(libs.androidx.constraintlayout)
+
+ implementation(libs.androidx.lifecycle.viewmodel.ktx)
+ implementation(libs.androidx.lifecycle.runtime.ktx)
+ implementation(libs.common.kotlin.coroutines.android)
+
+ implementation(libs.koin.android)
+
+ implementation("org.tensorflow:tensorflow-lite:2.14.0")
+ implementation("org.tensorflow:tensorflow-lite-support:0.4.4")
+ implementation("org.tensorflow:tensorflow-lite-gpu:2.14.0")
+
+ implementation("com.google.mlkit:text-recognition:16.0.0")
+
+ testImplementation(libs.tests.junit)
+ androidTestImplementation(libs.tests.androidx.junit)
+ androidTestImplementation(libs.tests.androidx.espresso.core)
+ implementation(projects.commonUi)
+}

cv-image-to-xml/src/androidTest/java/com/example/images/ExampleInstrumentedTest.kt

@@ -0,0 +1,24 @@
+package com.example.images
+
+import androidx.test.platform.app.InstrumentationRegistry
+import androidx.test.ext.junit.runners.AndroidJUnit4
+
+import org.junit.Test
+import org.junit.runner.RunWith
+
+import org.junit.Assert.*
+
+/**
+ * Instrumented test, which will execute on an Android device.
+ *
+ * See [testing documentation](http://d.android.com/tools/testing).
+ */
+@RunWith(AndroidJUnit4::class)
+class ExampleInstrumentedTest {
+ @Test
+ fun useAppContext() {
+ // Context of the app under test.
+ val appContext = InstrumentationRegistry.getInstrumentation().targetContext
+ assertEquals("com.example.images", appContext.packageName)
+ }
+}

cv-image-to-xml/src/main/AndroidManifest.xml

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+ xmlns:tools="http://schemas.android.com/tools">
+
+ <uses-feature
+ android:name="android.hardware.camera"
+ android:required="true" />
+
+ <uses-permission android:name="android.permission.CAMERA" />
+
+ <application
+ android:allowBackup="true"
+ android:dataExtractionRules="@xml/data_extraction_rules"
+ android:fullBackupContent="@xml/backup_rules"
+ android:icon="@mipmap/ic_launcher"
+ android:label="@string/app_name"
+ android:roundIcon="@mipmap/ic_launcher_round"
+ android:supportsRtl="true"
+ android:theme="@style/Theme.Images"
+ tools:targetApi="31">
+ <activity
+ android:name=".ui.ComputerVisionActivity"
+ android:exported="true"/>
+ </application>
+</manifest>

cv-image-to-xml/src/main/assets/labels.txt

@@ -0,0 +1,11 @@
+dropdown
+image_placeholder
+button
+checkbox_checked
+radio_button_unchecked
+radio_button_checked
+text_entry_box
+checkbox_unchecked
+slider
+switch_off
+switch_on

cv-image-to-xml/src/main/java/org/appdevforall/codeonthego/computervision/data/repository/ComputerVisionRepository.kt

@@ -0,0 +1,32 @@
+package org.appdevforall.codeonthego.computervision.data.repository
+import android.graphics.Bitmap
+import org.appdevforall.codeonthego.computervision.domain.model.DetectionResult
+import com.google.mlkit.vision.text.Text
+
+interface ComputerVisionRepository {
+
+ suspend fun initializeModel(): Result<Unit>
+
+ suspend fun runYoloInference(bitmap: Bitmap): Result<List<DetectionResult>>
+
+ suspend fun runOcrRecognition(bitmap: Bitmap): Result<List<Text.TextBlock>>
+
+ suspend fun mergeDetections(
+ yoloDetections: List<DetectionResult>,
+ textBlocks: List<Text.TextBlock>
+ ): Result<List<DetectionResult>>
+
+ suspend fun generateXml(
+ detections: List<DetectionResult>,
+ sourceImageWidth: Int,
+ sourceImageHeight: Int,
+ targetDpWidth: Int = 360,
+ targetDpHeight: Int = 640
+ ): Result<String>
+
+ fun preprocessBitmapForOcr(bitmap: Bitmap): Bitmap
+
+ fun isModelInitialized(): Boolean
+
+ fun releaseResources()
+}

cv-image-to-xml/src/main/java/org/appdevforall/codeonthego/computervision/data/repository/ComputerVisionRepositoryImpl.kt

@@ -0,0 +1,85 @@
+package org.appdevforall.codeonthego.computervision.data.repository
+
+import android.content.res.AssetManager
+import android.graphics.Bitmap
+import org.appdevforall.codeonthego.computervision.domain.DetectionMerger
+import org.appdevforall.codeonthego.computervision.domain.YoloToXmlConverter
+import org.appdevforall.codeonthego.computervision.data.source.OcrSource
+import org.appdevforall.codeonthego.computervision.data.source.YoloModelSource
+import org.appdevforall.codeonthego.computervision.domain.model.DetectionResult
+import org.appdevforall.codeonthego.computervision.utils.BitmapUtils
+import org.appdevforall.codeonthego.computervision.utils.TextCleaner
+import com.google.mlkit.vision.text.Text
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.withContext
+
+class ComputerVisionRepositoryImpl(
+ private val assetManager: AssetManager,
+ private val yoloModelSource: YoloModelSource,
+ private val ocrSource: OcrSource
+) : ComputerVisionRepository {
+
+ override suspend fun initializeModel(): Result<Unit> = withContext(Dispatchers.IO) {
+ runCatching {
+ yoloModelSource.initialize(assetManager)
+ }
+ }
+
+ override suspend fun runYoloInference(bitmap: Bitmap): Result<List<DetectionResult>> =
+ withContext(Dispatchers.Default) {
+ runCatching {
+ yoloModelSource.runInference(bitmap)
+ }
+ }
+
+ override suspend fun runOcrRecognition(bitmap: Bitmap): Result<List<Text.TextBlock>> =
+ withContext(Dispatchers.IO) {
+ ocrSource.recognizeText(bitmap)
+ }
+
+ override suspend fun mergeDetections(
+ yoloDetections: List<DetectionResult>,
+ textBlocks: List<Text.TextBlock>
+ ): Result<List<DetectionResult>> = withContext(Dispatchers.Default) {
+ runCatching {
+ val merger = DetectionMerger(yoloDetections, textBlocks)
+ val merged = merger.merge()
+
+ merged.forEach { detection ->
+ if (detection.text.isNotEmpty()) {
+ detection.text = TextCleaner.cleanText(detection.text)
+ }
+ }
+
+ merged
+ }
+ }
+
+ override suspend fun generateXml(
+ detections: List<DetectionResult>,
+ sourceImageWidth: Int,
+ sourceImageHeight: Int,
+ targetDpWidth: Int,
+ targetDpHeight: Int
+ ): Result<String> = withContext(Dispatchers.Default) {
+ runCatching {
+ YoloToXmlConverter.generateXmlLayout(
+ detections = detections,
+ sourceImageWidth = sourceImageWidth,
+ sourceImageHeight = sourceImageHeight,
+ targetDpWidth = targetDpWidth,
+ targetDpHeight = targetDpHeight
+ )
+ }
+ }
+
+ override fun preprocessBitmapForOcr(bitmap: Bitmap): Bitmap {
+ return BitmapUtils.preprocessForOcr(bitmap)
+ }
+
+ override fun isModelInitialized(): Boolean = yoloModelSource.isInitialized()
+
+ override fun releaseResources() {
+ yoloModelSource.release()
+ }
+}