From 20c7490558ba18bbdb150fb02272e76f344d936f Mon Sep 17 00:00:00 2001
From: Walter Perdan <github@kalwaltart.it>
Date: Sat, 30 May 2026 18:31:26 +0200
Subject: [PATCH] Fix crash when a marker is detected on the first frame (#37)

processFrame() could call GetTrackedFeaturesWarped() with an empty tracked-
point selection on the very first frame: the optical-flow branch that calls
GetInitialFeatures() (which populates _selectedPts) is gated on
_frameCount > 0, yet MatchFeatures() can set _isDetected = true on frame 0.
cv::perspectiveTransform then asserts on the empty input (scn+1 == m.cols,
i.e. 2 == 3) and throws.

Two guards:
- TrackingPointSelector::GetTrackedFeaturesWarped() returns an empty result
  when the selection is empty or the homography is empty, instead of calling
  perspectiveTransform on empty input.
- processFrame()'s pose block only runs solvePnP when there are >= 4
  correspondences and the image/object point counts match.

A live camera masks this (detection essentially never succeeds on the
literal first frame); a static image triggers it deterministically. The
webarkit-testing static example previously worked around it with a blank
warmup frame; this fixes it at the source.

Refs: webarkit/WebARKitLib#37, webarkit/webarkit-testing#30

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 .../TrackingPointSelector.cpp                 |  7 +++++++
 .../WebARKitTracker.cpp                       | 21 ++++++++++++-------
 2 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/TrackingPointSelector.cpp b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/TrackingPointSelector.cpp
index e77a5fc..8694c83 100644
--- a/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/TrackingPointSelector.cpp
+++ b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/TrackingPointSelector.cpp
@@ -170,6 +170,13 @@ std::vector<cv::Point2f> TrackingPointSelector::GetTrackedFeaturesWarped()
 {
     std::vector<cv::Point2f> selectedPoints = GetTrackedFeatures();
     std::vector<cv::Point2f> warpedPoints;
+    // cv::perspectiveTransform asserts on an empty input (and requires a valid
+    // 3x3 homography). This happens when a marker is detected on the very first
+    // frame, before the tracked-point selection has been populated. Return an
+    // empty result instead of throwing.
+    if (selectedPoints.empty() || _homography.empty()) {
+        return warpedPoints;
+    }
     perspectiveTransform(selectedPoints, warpedPoints, _homography);
     return warpedPoints;
 }
diff --git a/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/WebARKitTracker.cpp b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/WebARKitTracker.cpp
index 2d6ed35..aa5f1bb 100644
--- a/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/WebARKitTracker.cpp
+++ b/WebARKit/WebARKitTrackers/WebARKitOpticalTracking/WebARKitTracker.cpp
@@ -390,13 +390,20 @@ class WebARKitTracker::WebARKitTrackerImpl {
             cv::Mat _pose;
             std::vector<cv::Point2f> imgPoints = _trackSelection.GetTrackedFeaturesWarped();
             std::vector<cv::Point3f> objPoints = _trackSelection.GetTrackedFeatures3d();
-            _patternTrackingInfo.cameraPoseFromPoints(_pose, objPoints, imgPoints, m_camMatrix, m_distortionCoeff);
-            // _patternTrackingInfo.computePose(_pattern.points3d, warpedCorners, m_camMatrix, m_distortionCoeff);
-            _patternTrackingInfo.getTrackablePose(_pose);
-            _patternTrackingInfo.updateTrackable();
-            _patternTrackingInfo.computeGLviewMatrix(_pose);
-            fill_output(m_H);
-            WEBARKIT_LOGi("Marker tracked ! Num. matches : %d\n", numMatches);
+            // Need at least 4 correspondences for solvePnP, and the two sets must
+            // match in size. On the very first frame a marker can be detected
+            // before the tracked-point selection is populated (the optical-flow
+            // branch that calls GetInitialFeatures() is gated on _frameCount > 0),
+            // leaving these empty. Skip pose estimation in that case.
+            if (imgPoints.size() >= 4 && imgPoints.size() == objPoints.size()) {
+                _patternTrackingInfo.cameraPoseFromPoints(_pose, objPoints, imgPoints, m_camMatrix, m_distortionCoeff);
+                // _patternTrackingInfo.computePose(_pattern.points3d, warpedCorners, m_camMatrix, m_distortionCoeff);
+                _patternTrackingInfo.getTrackablePose(_pose);
+                _patternTrackingInfo.updateTrackable();
+                _patternTrackingInfo.computeGLviewMatrix(_pose);
+                fill_output(m_H);
+                WEBARKIT_LOGi("Marker tracked ! Num. matches : %d\n", numMatches);
+            }
         }
 
         swapImagePyramid();