Stop using pyrUp in fastMatchTemplate

It's not clear that using pyrUp actually gives better results. Our match data is necessarily quite spiky, and the gaussian convolution blurs the results too much. This causes us to restrict too much the mask for the matchTemplate on the subsequent layers.
efokschaner · Jan 13, 2016 · 66c2a2c · 66c2a2c
1 parent 0844ca4
commit 66c2a2c
Show file tree

Hide file tree

Showing 2 changed files with 73 additions and 41 deletions.
diff --git a/app/src/main/java/efokschaner/infinityloopsolver/Debug.java b/app/src/main/java/efokschaner/infinityloopsolver/Debug.java
@@ -43,45 +43,49 @@ public static void sendBitmap(Bitmap bitmap) {
         mExecutor.submit(new Runnable() {
             @Override
             public void run() {
+                sendBitmapSync(copy);
+            }
+        });
+    }
+
+    public static void sendBitmapSync(Bitmap bitmap) {
+        try {
+            URL url = new URL("http://192.168.0.5:8888/" + getNewTimestamp() + ".png");
+            try {
+                HttpURLConnection conn = (HttpURLConnection) url.openConnection();
                 try {
-                    URL url = new URL("http://efoks1ml1:8888/" + getNewTimestamp() + ".png");
-                    try {
-                        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
-                        try {
-                            conn.setDoOutput(true);
-                            conn.setChunkedStreamingMode(0);
-                            conn.setRequestMethod("POST");
-                            conn.setRequestProperty("Content-Type", "application/octet-stream");
-                            try (OutputStream ostream = conn.getOutputStream()) {
-                                copy.compress(Bitmap.CompressFormat.PNG, 100, ostream);
-                            } catch (IOException e) {
-                                e.printStackTrace();
-                            }
-                            final int responseCode = conn.getResponseCode();
-                            if (!(responseCode >= 200 && responseCode < 300)) {
-                                throw new AssertionError(String.format("Http response was: %d", responseCode));
-                            }
-                            conn.getResponseMessage();
-                            BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream()));
-                            while (in.readLine() != null){
-                                // ignore contents
-                            }
-                            in.close();
-                        } catch (ProtocolException e) {
-                            e.printStackTrace();
-                        } finally {
-                            conn.disconnect();
-                        }
+                    conn.setDoOutput(true);
+                    conn.setChunkedStreamingMode(0);
+                    conn.setRequestMethod("POST");
+                    conn.setRequestProperty("Content-Type", "application/octet-stream");
+                    try (OutputStream ostream = conn.getOutputStream()) {
+                        bitmap.compress(Bitmap.CompressFormat.PNG, 100, ostream);
                     } catch (IOException e) {
                         e.printStackTrace();
                     }
-                } catch (MalformedURLException e) {
+                    final int responseCode = conn.getResponseCode();
+                    if (!(responseCode >= 200 && responseCode < 300)) {
+                        throw new AssertionError(String.format("Http response was: %d", responseCode));
+                    }
+                    conn.getResponseMessage();
+                    BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream()));
+                    while (in.readLine() != null){
+                        // ignore contents
+                    }
+                    in.close();
+                } catch (ProtocolException e) {
                     e.printStackTrace();
                 } finally {
-                    copy.recycle();
+                    conn.disconnect();
                 }
+            } catch (IOException e) {
+                e.printStackTrace();
             }
-        });
+        } catch (MalformedURLException e) {
+            e.printStackTrace();
+        } finally {
+            bitmap.recycle();
+        }
     }
 
     public static void sendMatrix(Mat m) {
@@ -91,6 +95,12 @@ public static void sendMatrix(Mat m) {
         b2.recycle();
     }
 
+    public static void sendMatrixSync(Mat m) {
+        Bitmap b2 = Bitmap.createBitmap(m.width(), m.height(), Bitmap.Config.ARGB_8888);
+        Utils.matToBitmap(m, b2);
+        sendBitmapSync(b2);
+    }
+
     public static void sendScreenshot(UiAutomation uiAutomation) {
         Bitmap b = uiAutomation.takeScreenshot();
         try{

diff --git a/app/src/main/java/efokschaner/infinityloopsolver/ImageProcessor.java b/app/src/main/java/efokschaner/infinityloopsolver/ImageProcessor.java
@@ -30,11 +30,17 @@ public class ImageProcessor {
     private static final boolean DEBUG = false;
     private static final boolean PROFILE = false;
     private static final int PYRAMID_LEVELS = 3;
+    private static final double TEMPLATE_MATCH_THRESH_VAL = 0.3;
     private static final FastMatchThresholdCallback SQDIFF_NORMED_FAST_MATCH_CALLBACK = new FastMatchThresholdCallback() {
         @Override
         public Mat call(Mat match) {
             Mat matchThreshed = new Mat();
-            Imgproc.threshold(match, matchThreshed, 0.75, 1, Imgproc.THRESH_BINARY_INV);
+            Imgproc.threshold(
+                    match,
+                    matchThreshed,
+                    0.75,
+                    255,
+                    Imgproc.THRESH_BINARY_INV);
             // Reset to 1 because with sqdiff algorithm zero corresponds to match
             match.setTo(new Scalar(1));
             return matchThreshed;
@@ -90,22 +96,30 @@ public static Mat fastMatchTemplate(
         for (int curLevel = maxLevel - 1; curLevel >= 0; --curLevel) {
             Mat scene = scenePyr.get(curLevel);
             Mat template = templatePyr.get(curLevel);
-            Mat prevMatchResultUp = new Mat();
-            Imgproc.pyrUp(prevMatchResult, prevMatchResultUp);
-            // prevMatchResultUp is conceptually an identical space to the new matchResult,
-            // but due to quantisation errors in the halving / doubling process it can be slightly
-            // different size. We'll resize it to be identical though as it allows for less
-            // defensive coding in the subsequent operations
+            if(DEBUG) {
+                Debug.sendMatrixSync(template);
+            }
             Mat prevMatchResultResized = new Mat();
             Size newMatchResultSize = new Size(
                     scene.width() - template.width() + 1,
                     scene.height() - template.height() + 1);
-            Imgproc.resize(prevMatchResultUp, prevMatchResultResized, newMatchResultSize);
+            // Don't bother with fancy pyrUp, its not clear that using pyr up to upsample the
+            // last layer's results actually improves accuracy.
+            Imgproc.resize(prevMatchResult, prevMatchResultResized, newMatchResultSize);
+            if(DEBUG) {
+                Mat debug = prevMatchResultResized.mul(Mat.ones(prevMatchResultResized.size(), prevMatchResultResized.type()), 255);
+                Mat debug2 = new Mat();
+                debug.convertTo(debug2, CvType.CV_8UC1);
+                Debug.sendMatrixSync(debug2);
+            }
             Mat prevMatchResultThreshed = cb.call(prevMatchResultResized);
             // Renaming for clarity as the callback should have reset the matrix
             Mat matchResult = prevMatchResultResized;
             Mat mask8u = new Mat();
             prevMatchResultThreshed.convertTo(mask8u, CvType.CV_8U);
+            if(DEBUG) {
+                Debug.sendMatrixSync(mask8u);
+            }
             List<MatOfPoint> contours = new ArrayList<>();
             Imgproc.findContours(
                     mask8u,
@@ -121,6 +135,9 @@ public static Mat fastMatchTemplate(
                         boundingRect.width + template.width() - 1,
                         boundingRect.height + template.height() - 1);
                 Mat sceneRoi = new Mat(scene, sceneRoiRect);
+                if(DEBUG) {
+                    Debug.sendMatrixSync(sceneRoi);
+                }
                 Imgproc.matchTemplate(
                         sceneRoi,
                         template,
@@ -297,7 +314,12 @@ public GameState getGameStateFromImage(Bitmap b) {
                             Imgproc.TM_SQDIFF_NORMED,
                             SQDIFF_NORMED_FAST_MATCH_CALLBACK);
                     Mat matchThreshed = new Mat();
-                    Imgproc.threshold(match, matchThreshed, 0.3, 255, Imgproc.THRESH_BINARY_INV);
+                    Imgproc.threshold(
+                            match,
+                            matchThreshed,
+                            TEMPLATE_MATCH_THRESH_VAL,
+                            255,
+                            Imgproc.THRESH_BINARY_INV);
                     Mat eightBitMatchThreshed = new Mat();
                     matchThreshed.convertTo(eightBitMatchThreshed, CvType.CV_8U);
                     if (DEBUG) {
@@ -571,7 +593,7 @@ private double getTileScale(ArrayList<Mat> gameImageRoiPyramid) {
                     bestScore = Math.min(bestScore, minMaxLocResult.minVal);
                 }
                 // Just in case there weren't any matches at all
-                if(bestScore < 0.2) {
+                if(bestScore < TEMPLATE_MATCH_THRESH_VAL) {
                     scaleScoreMapping.put(tileScaleEntry.getKey(), bestScore);
                 }
             }