From 66c2a2cbf9388b12db149c063759bcac85d2d2af Mon Sep 17 00:00:00 2001 From: efokschaner Date: Wed, 13 Jan 2016 01:10:31 -0800 Subject: [PATCH] Stop using pyrUp in fastMatchTemplate It's not clear that using pyrUp actually gives better results. Our match data is necessarily quite spiky, and the gaussian convolution blurs the results too much. This causes us to restrict too much the mask for the matchTemplate on the subsequent layers. --- .../efokschaner/infinityloopsolver/Debug.java | 72 +++++++++++-------- .../infinityloopsolver/ImageProcessor.java | 42 ++++++++--- 2 files changed, 73 insertions(+), 41 deletions(-) diff --git a/app/src/main/java/efokschaner/infinityloopsolver/Debug.java b/app/src/main/java/efokschaner/infinityloopsolver/Debug.java index 1d9829b..e07bb7c 100644 --- a/app/src/main/java/efokschaner/infinityloopsolver/Debug.java +++ b/app/src/main/java/efokschaner/infinityloopsolver/Debug.java @@ -43,45 +43,49 @@ public static void sendBitmap(Bitmap bitmap) { mExecutor.submit(new Runnable() { @Override public void run() { + sendBitmapSync(copy); + } + }); + } + + public static void sendBitmapSync(Bitmap bitmap) { + try { + URL url = new URL("http://192.168.0.5:8888/" + getNewTimestamp() + ".png"); + try { + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); try { - URL url = new URL("http://efoks1ml1:8888/" + getNewTimestamp() + ".png"); - try { - HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - try { - conn.setDoOutput(true); - conn.setChunkedStreamingMode(0); - conn.setRequestMethod("POST"); - conn.setRequestProperty("Content-Type", "application/octet-stream"); - try (OutputStream ostream = conn.getOutputStream()) { - copy.compress(Bitmap.CompressFormat.PNG, 100, ostream); - } catch (IOException e) { - e.printStackTrace(); - } - final int responseCode = conn.getResponseCode(); - if (!(responseCode >= 200 && responseCode < 300)) { - throw new AssertionError(String.format("Http response was: %d", responseCode)); - } - conn.getResponseMessage(); - BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream())); - while (in.readLine() != null){ - // ignore contents - } - in.close(); - } catch (ProtocolException e) { - e.printStackTrace(); - } finally { - conn.disconnect(); - } + conn.setDoOutput(true); + conn.setChunkedStreamingMode(0); + conn.setRequestMethod("POST"); + conn.setRequestProperty("Content-Type", "application/octet-stream"); + try (OutputStream ostream = conn.getOutputStream()) { + bitmap.compress(Bitmap.CompressFormat.PNG, 100, ostream); } catch (IOException e) { e.printStackTrace(); } - } catch (MalformedURLException e) { + final int responseCode = conn.getResponseCode(); + if (!(responseCode >= 200 && responseCode < 300)) { + throw new AssertionError(String.format("Http response was: %d", responseCode)); + } + conn.getResponseMessage(); + BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream())); + while (in.readLine() != null){ + // ignore contents + } + in.close(); + } catch (ProtocolException e) { e.printStackTrace(); } finally { - copy.recycle(); + conn.disconnect(); } + } catch (IOException e) { + e.printStackTrace(); } - }); + } catch (MalformedURLException e) { + e.printStackTrace(); + } finally { + bitmap.recycle(); + } } public static void sendMatrix(Mat m) { @@ -91,6 +95,12 @@ public static void sendMatrix(Mat m) { b2.recycle(); } + public static void sendMatrixSync(Mat m) { + Bitmap b2 = Bitmap.createBitmap(m.width(), m.height(), Bitmap.Config.ARGB_8888); + Utils.matToBitmap(m, b2); + sendBitmapSync(b2); + } + public static void sendScreenshot(UiAutomation uiAutomation) { Bitmap b = uiAutomation.takeScreenshot(); try{ diff --git a/app/src/main/java/efokschaner/infinityloopsolver/ImageProcessor.java b/app/src/main/java/efokschaner/infinityloopsolver/ImageProcessor.java index 4c8f156..1e3e219 100644 --- a/app/src/main/java/efokschaner/infinityloopsolver/ImageProcessor.java +++ b/app/src/main/java/efokschaner/infinityloopsolver/ImageProcessor.java @@ -30,11 +30,17 @@ public class ImageProcessor { private static final boolean DEBUG = false; private static final boolean PROFILE = false; private static final int PYRAMID_LEVELS = 3; + private static final double TEMPLATE_MATCH_THRESH_VAL = 0.3; private static final FastMatchThresholdCallback SQDIFF_NORMED_FAST_MATCH_CALLBACK = new FastMatchThresholdCallback() { @Override public Mat call(Mat match) { Mat matchThreshed = new Mat(); - Imgproc.threshold(match, matchThreshed, 0.75, 1, Imgproc.THRESH_BINARY_INV); + Imgproc.threshold( + match, + matchThreshed, + 0.75, + 255, + Imgproc.THRESH_BINARY_INV); // Reset to 1 because with sqdiff algorithm zero corresponds to match match.setTo(new Scalar(1)); return matchThreshed; @@ -90,22 +96,30 @@ public static Mat fastMatchTemplate( for (int curLevel = maxLevel - 1; curLevel >= 0; --curLevel) { Mat scene = scenePyr.get(curLevel); Mat template = templatePyr.get(curLevel); - Mat prevMatchResultUp = new Mat(); - Imgproc.pyrUp(prevMatchResult, prevMatchResultUp); - // prevMatchResultUp is conceptually an identical space to the new matchResult, - // but due to quantisation errors in the halving / doubling process it can be slightly - // different size. We'll resize it to be identical though as it allows for less - // defensive coding in the subsequent operations + if(DEBUG) { + Debug.sendMatrixSync(template); + } Mat prevMatchResultResized = new Mat(); Size newMatchResultSize = new Size( scene.width() - template.width() + 1, scene.height() - template.height() + 1); - Imgproc.resize(prevMatchResultUp, prevMatchResultResized, newMatchResultSize); + // Don't bother with fancy pyrUp, its not clear that using pyr up to upsample the + // last layer's results actually improves accuracy. + Imgproc.resize(prevMatchResult, prevMatchResultResized, newMatchResultSize); + if(DEBUG) { + Mat debug = prevMatchResultResized.mul(Mat.ones(prevMatchResultResized.size(), prevMatchResultResized.type()), 255); + Mat debug2 = new Mat(); + debug.convertTo(debug2, CvType.CV_8UC1); + Debug.sendMatrixSync(debug2); + } Mat prevMatchResultThreshed = cb.call(prevMatchResultResized); // Renaming for clarity as the callback should have reset the matrix Mat matchResult = prevMatchResultResized; Mat mask8u = new Mat(); prevMatchResultThreshed.convertTo(mask8u, CvType.CV_8U); + if(DEBUG) { + Debug.sendMatrixSync(mask8u); + } List contours = new ArrayList<>(); Imgproc.findContours( mask8u, @@ -121,6 +135,9 @@ public static Mat fastMatchTemplate( boundingRect.width + template.width() - 1, boundingRect.height + template.height() - 1); Mat sceneRoi = new Mat(scene, sceneRoiRect); + if(DEBUG) { + Debug.sendMatrixSync(sceneRoi); + } Imgproc.matchTemplate( sceneRoi, template, @@ -297,7 +314,12 @@ public GameState getGameStateFromImage(Bitmap b) { Imgproc.TM_SQDIFF_NORMED, SQDIFF_NORMED_FAST_MATCH_CALLBACK); Mat matchThreshed = new Mat(); - Imgproc.threshold(match, matchThreshed, 0.3, 255, Imgproc.THRESH_BINARY_INV); + Imgproc.threshold( + match, + matchThreshed, + TEMPLATE_MATCH_THRESH_VAL, + 255, + Imgproc.THRESH_BINARY_INV); Mat eightBitMatchThreshed = new Mat(); matchThreshed.convertTo(eightBitMatchThreshed, CvType.CV_8U); if (DEBUG) { @@ -571,7 +593,7 @@ private double getTileScale(ArrayList gameImageRoiPyramid) { bestScore = Math.min(bestScore, minMaxLocResult.minVal); } // Just in case there weren't any matches at all - if(bestScore < 0.2) { + if(bestScore < TEMPLATE_MATCH_THRESH_VAL) { scaleScoreMapping.put(tileScaleEntry.getKey(), bestScore); } }