ADFA-3108: Improve CV-to-XML accuracy with fuzzy search and OCR refinement

cv-image-to-xml/build.gradle.kts

@@ -44,6 +44,8 @@ dependencies {
 
  implementation("com.google.mlkit:text-recognition:16.0.1")
 
+ implementation(libs.composite.fuzzysearch)
+
  testImplementation(libs.tests.junit)
  androidTestImplementation(libs.tests.androidx.junit)
  androidTestImplementation(libs.tests.androidx.espresso.core)

cv-image-to-xml/src/main/java/org/appdevforall/codeonthego/computervision/data/repository/ComputerVisionRepository.kt

@@ -2,6 +2,7 @@ package org.appdevforall.codeonthego.computervision.data.repository
 
 import android.graphics.Bitmap
 import com.google.mlkit.vision.text.Text
+import org.appdevforall.codeonthego.computervision.domain.RegionOcrProcessor
 import org.appdevforall.codeonthego.computervision.domain.model.DetectionResult
 
 interface ComputerVisionRepository {
@@ -10,11 +11,17 @@ interface ComputerVisionRepository {
 
  suspend fun runYoloInference(bitmap: Bitmap): Result<List<DetectionResult>>
 
- suspend fun runOcrRecognition(bitmap: Bitmap): Result<List<Text.TextBlock>>
+ suspend fun runRegionOcr(
+ bitmap: Bitmap,
+ yoloDetections: List<DetectionResult>,
+ leftGuidePct: Float,
+ rightGuidePct: Float
+ ): Result<RegionOcrProcessor.RegionOcrResult>
 
  suspend fun mergeDetections(
- yoloDetections: List<DetectionResult>,
- textBlocks: List<Text.TextBlock>
+ enrichedComponents: List<DetectionResult>,
+ remainingDetections: List<DetectionResult>,
+ fullImageTextBlocks: List<Text.TextBlock>
  ): Result<List<DetectionResult>>
 
  suspend fun generateXml(
@@ -26,9 +33,7 @@ interface ComputerVisionRepository {
  targetDpHeight: Int
  ): Result<String>
 
- fun preprocessBitmapForOcr(bitmap: Bitmap): Bitmap
-
  fun isModelInitialized(): Boolean
 
  fun releaseResources()
-}
+}

cv-image-to-xml/src/main/java/org/appdevforall/codeonthego/computervision/data/repository/ComputerVisionRepositoryImpl.kt

@@ -3,19 +3,18 @@ package org.appdevforall.codeonthego.computervision.data.repository
 import android.content.res.AssetManager
 import android.graphics.Bitmap
 import org.appdevforall.codeonthego.computervision.domain.DetectionMerger
+import org.appdevforall.codeonthego.computervision.domain.RegionOcrProcessor
 import org.appdevforall.codeonthego.computervision.domain.YoloToXmlConverter
-import org.appdevforall.codeonthego.computervision.data.source.OcrSource
 import org.appdevforall.codeonthego.computervision.data.source.YoloModelSource
 import org.appdevforall.codeonthego.computervision.domain.model.DetectionResult
-import org.appdevforall.codeonthego.computervision.utils.BitmapUtils
 import com.google.mlkit.vision.text.Text
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.withContext
 
 class ComputerVisionRepositoryImpl(
  private val assetManager: AssetManager,
  private val yoloModelSource: YoloModelSource,
- private val ocrSource: OcrSource
+ private val regionOcrProcessor: RegionOcrProcessor
 ) : ComputerVisionRepository {
 
  override suspend fun initializeModel(): Result<Unit> = withContext(Dispatchers.IO) {
@@ -31,18 +30,24 @@ class ComputerVisionRepositoryImpl(
  }
  }
 
- override suspend fun runOcrRecognition(bitmap: Bitmap): Result<List<Text.TextBlock>> =
- withContext(Dispatchers.IO) {
- ocrSource.recognizeText(bitmap)
+ override suspend fun runRegionOcr(
+ bitmap: Bitmap,
+ yoloDetections: List<DetectionResult>,
+ leftGuidePct: Float,
+ rightGuidePct: Float
+ ): Result<RegionOcrProcessor.RegionOcrResult> = withContext(Dispatchers.Default) {
+ runCatching {
+ regionOcrProcessor.process(bitmap, yoloDetections, leftGuidePct, rightGuidePct)
  }
+ }
 
  override suspend fun mergeDetections(
- yoloDetections: List<DetectionResult>,
- textBlocks: List<Text.TextBlock>
+ enrichedComponents: List<DetectionResult>,
+ remainingDetections: List<DetectionResult>,
+ fullImageTextBlocks: List<Text.TextBlock>
  ): Result<List<DetectionResult>> = withContext(Dispatchers.Default) {
  runCatching {
- val merger = DetectionMerger(yoloDetections, textBlocks)
- merger.merge()
+ DetectionMerger(enrichedComponents, remainingDetections, fullImageTextBlocks).merge()
  }
  }
 
@@ -66,13 +71,9 @@ class ComputerVisionRepositoryImpl(
  }
  }
 
- override fun preprocessBitmapForOcr(bitmap: Bitmap): Bitmap {
- return BitmapUtils.preprocessForOcr(bitmap)
- }
-
  override fun isModelInitialized(): Boolean = yoloModelSource.isInitialized()
 
  override fun releaseResources() {
  yoloModelSource.release()
  }
-}
+}

cv-image-to-xml/src/main/java/org/appdevforall/codeonthego/computervision/data/source/OcrSource.kt

@@ -1,11 +1,14 @@
 package org.appdevforall.codeonthego.computervision.data.source
+
 import android.graphics.Bitmap
+import android.util.Log
 import com.google.mlkit.vision.common.InputImage
 import com.google.mlkit.vision.text.Text
 import com.google.mlkit.vision.text.TextRecognition
 import com.google.mlkit.vision.text.latin.TextRecognizerOptions
 import kotlinx.coroutines.suspendCancellableCoroutine
 import kotlin.coroutines.resume
+
 class OcrSource {
 
  private val textRecognizer by lazy {

cv-image-to-xml/src/main/java/org/appdevforall/codeonthego/computervision/di/ComputerVisionModule.kt

@@ -1,8 +1,10 @@
 package org.appdevforall.codeonthego.computervision.di
+
 import org.appdevforall.codeonthego.computervision.data.repository.ComputerVisionRepository
 import org.appdevforall.codeonthego.computervision.data.repository.ComputerVisionRepositoryImpl
 import org.appdevforall.codeonthego.computervision.data.source.OcrSource
 import org.appdevforall.codeonthego.computervision.data.source.YoloModelSource
+import org.appdevforall.codeonthego.computervision.domain.RegionOcrProcessor
 import org.appdevforall.codeonthego.computervision.ui.viewmodel.ComputerVisionViewModel
 import org.koin.android.ext.koin.androidContext
 import org.koin.core.module.dsl.viewModel
@@ -14,11 +16,13 @@ val computerVisionModule = module {
 
  single { OcrSource() }
 
+ single { RegionOcrProcessor(ocrSource = get()) }
+
  single<ComputerVisionRepository> {
  ComputerVisionRepositoryImpl(
  assetManager = androidContext().assets,
  yoloModelSource = get(),
-ocrSource = get()
+regionOcrProcessor = get()
  )
  }
 

cv-image-to-xml/src/main/java/org/appdevforall/codeonthego/computervision/domain/DetectionMerger.kt

@@ -4,54 +4,27 @@ import android.graphics.RectF
 import com.google.mlkit.vision.text.Text
 import org.appdevforall.codeonthego.computervision.domain.model.DetectionResult
 
-/**
- * A dedicated, robust class to safely merge YOLO detections with ML Kit text results.
- * This class uses a crash-proof, multi-pass filtering approach that is immune to
- * ConcurrentModificationExceptions.
- */
 class DetectionMerger(
- private val yoloResults: List<DetectionResult>,
- private val allTextBlocks: List<Text.TextBlock>
+ private val enrichedComponents: List<DetectionResult>,
+ private val remainingYoloDetections: List<DetectionResult>,
+ private val fullImageTextBlocks: List<Text.TextBlock>
 ) {
 
- // Define the roles for different labels for hierarchical processing
- private val componentLabels = setOf("button", "checkbox_checked", "checkbox_unchecked", "switch_on", "switch_off", "chip", "text_entry_box")
  private val containerLabels = setOf("card", "toolbar")
 
- /**
- * Executes the merging logic by running a series of ordered, non-destructive passes.
- * @return The final, correctly merged list of all UI detections.
- */
  fun merge(): List<DetectionResult> {
-
- // Pass 1: Interactive components get exclusive first rights to claim text.
- // This returns a map of which text block is now "reserved" by which component.
- val componentTextClaims = claimTextForComponents()
-
- // Pass 2: Build the final list, respecting the claims made in Pass 1.
  val finalDetections = mutableListOf<DetectionResult>()
  val usedTextBlocks = mutableSetOf<Text.TextBlock>()
 
- // Add all YOLO results, enriching them with their claimed text.
- for (yoloResult in yoloResults) {
- val claimedText = componentTextClaims[yoloResult]
- if (claimedText != null) {
- yoloResult.text = claimedText.text.replace("\n", " ")
- usedTextBlocks.add(claimedText)
- }
- finalDetections.add(yoloResult)
- }
-
- // Pass 3: Process containers and add their *unclaimed* internal content.
- val yoloContainers = yoloResults.filter { it.label in containerLabels }
- for (container in yoloContainers) {
- val contentCandidates = allTextBlocks.filter { it !in usedTextBlocks }
+ finalDetections.addAll(enrichedComponents)
+ finalDetections.addAll(remainingYoloDetections)
 
- for (textBlock in contentCandidates) {
+ val containers = remainingYoloDetections.filter { it.label in containerLabels }
+ for (container in containers) {
+ val candidates = fullImageTextBlocks.filter { it !in usedTextBlocks }
+ for (textBlock in candidates) {
  val textBox = textBlock.boundingBox?.let { RectF(it) } ?: continue
  if (container.boundingBox.contains(textBox)) {
- // This text is inside the container and was not claimed by a button,
- // so it must be a standalone TextView within the card.
  finalDetections.add(
  DetectionResult(
  boundingBox = textBox,
@@ -61,14 +34,12 @@ class DetectionMerger(
  isYolo = false
  )
  )
- // Mark it as used so it doesn't also become an orphan.
  usedTextBlocks.add(textBlock)
  }
  }
  }
 
- // Pass 4: Any text that is still unused after all other passes is a true orphan.
- val orphanText = allTextBlocks.filter { it !in usedTextBlocks }
+ val orphanText = fullImageTextBlocks.filter { it !in usedTextBlocks }
  for (textBlock in orphanText) {
  textBlock.boundingBox?.let {
  finalDetections.add(
@@ -85,51 +56,4 @@ class DetectionMerger(
 
  return finalDetections
  }
-
- /**
- * Finds the single best text block for each interactive component based on IoU score.
- * This method is non-destructive and only returns a "plan" of claims.
- * @return A map where the key is the component and the value is the text block it has claimed.
- */
- private fun claimTextForComponents(): Map<DetectionResult, Text.TextBlock> {
- val claims = mutableMapOf<DetectionResult, Text.TextBlock>()
- val yoloComponents = yoloResults.filter { it.label in componentLabels }
- val availableText = allTextBlocks.toMutableSet() // Use a set for efficient removal
-
- for (component in yoloComponents) {
- val bestMatch = availableText
- .mapNotNull { textBlock ->
- textBlock.boundingBox?.let { box ->
- val iou = calculateIoU(component.boundingBox, RectF(box))
- Triple(textBlock, iou, component)
- }
- }
- .filter { it.second > 0.05 } // Must have at least some meaningful overlap
- .maxByOrNull { it.second } // Get the best match based on IoU
-
- // If a best match was found, reserve it and remove it from future consideration.
- bestMatch?.let { (textBlock, _, matchedComponent) ->
- claims[matchedComponent] = textBlock
- availableText.remove(textBlock)
- }
- }
- return claims
- }
-
- /**
- * Utility function to calculate Intersection over Union (IoU).
- */
- private fun calculateIoU(box1: RectF, box2: RectF): Float {
- val xA = maxOf(box1.left, box2.left)
- val yA = maxOf(box1.top, box2.top)
- val xB = minOf(box1.right, box2.right)
- val yB = minOf(box1.bottom, box2.bottom)
-
- val intersectionArea = maxOf(0f, xB - xA) * maxOf(0f, yB - yA)
- val box1Area = box1.width() * box1.height()
- val box2Area = box2.width() * box2.height()
- val unionArea = box1Area + box2Area - intersectionArea
-
- return if (unionArea == 0f) 0f else intersectionArea / unionArea
- }
-}
+}