diff --git a/.gitignore b/.gitignore index 80a6566..ada93c2 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,7 @@ proguard/ # Log Files *.log +*.trace # Dreaded DS_Store .DS_Store diff --git a/README.md b/README.md index cab539a..80b6577 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,11 @@ Make sure to pick the right architecture apk for the target device ### Launch the application: - adb shell am instrument -w efokschaner.infinityloopsolver/efokschaner.infinityloopsolver.SolverInstrumentation + adb shell am instrument -w efokschaner.infinityloopsolver/.SolverInstrumentation The application only works when launched this way as it uses UiAutomation api's that are not available to normally launched applications. + +*Protip:* Use `nohup` to let the application stay running when adb is disconnected: + adb shell 'nohup am instrument -w efokschaner.infinityloopsolver/.SolverInstrumentation &1 >/dev/null' + ^C (stopping adb doesn't break the app) \ No newline at end of file diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 4353bfc..c67f501 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -3,6 +3,7 @@ package="efokschaner.infinityloopsolver"> + mTileImages = new HashMap<>(); private static double globalScaleFactor = 0.5; private final ArrayList tileImageScalesRange = new ArrayList<>(); + // cv::BuildPyramid from Imgproc + public static void buildPyramid(Mat src, List dst, int maxlevel, int borderType) { + if(BuildConfig.DEBUG) { + if(borderType == Core.BORDER_CONSTANT) { + throw new AssertionError("No support for Core.BORDER_CONSTANT"); + } + } + dst.clear(); + dst.add(src.clone()); + Mat prevLevel = dst.get(0); + for (int i = 1; i <= maxlevel; ++i) { + Mat nextLevel = new Mat(); + Imgproc.pyrDown(prevLevel, nextLevel, new Size(), borderType); + dst.add(nextLevel); + prevLevel = nextLevel; + } + } + + // cv::BuildPyramid from Imgproc + public static void buildPyramid(Mat src, List dst, int maxlevel) { + buildPyramid(src, dst, maxlevel, Core.BORDER_DEFAULT); + } + + + // This callback should do two things. + // 1. Return a thresholded copy of match that masks the region to check on the next iteration + // 2. Reset the match Mat so that it possible to accumulate fresh matchResults into it. + // This may involve zeroing it out or filling it with large vals depending on your choice + // of matchTemplate method / thresholds etc. + public interface FastMatchThresholdCallback { + Mat call(Mat match); + } + + public static Mat fastMatchTemplate( + List scenePyr, + List templatePyr, + int method, + FastMatchThresholdCallback cb) { + final int maxLevel = Math.min(scenePyr.size(), templatePyr.size()) - 1; + Mat prevMatchResult = new Mat(); + Imgproc.matchTemplate(scenePyr.get(maxLevel), templatePyr.get(maxLevel), prevMatchResult, method); + for (int curLevel = maxLevel - 1; curLevel >= 0; --curLevel) { + Mat scene = scenePyr.get(curLevel); + Mat template = templatePyr.get(curLevel); + Mat prevMatchResultUp = new Mat(); + Imgproc.pyrUp(prevMatchResult, prevMatchResultUp); + // prevMatchResult is conceptually an identical space to the new matchResult, + // but due to quantisation errors in the halving / doubling process it can be slightly + // different size. We'll resize it to be identical though as it allows for less + // defensive coding in the subsequent operations + Mat prevMatchResultResized = new Mat(); + Size newMatchResultSize = new Size( + scene.width() - template.width() + 1, + scene.height() - template.height() + 1); + Imgproc.resize(prevMatchResult, prevMatchResultResized, newMatchResultSize); + Mat prevMatchResultThreshed = cb.call(prevMatchResultResized); + // Renaming for clarity as the callback should have reset the matrix + Mat matchResult = prevMatchResultResized; + Mat mask8u = new Mat(); + prevMatchResultThreshed.convertTo(mask8u, CvType.CV_8U); + List contours = new ArrayList<>(); + Imgproc.findContours( + mask8u, + contours, + new Mat(), + Imgproc.RETR_EXTERNAL, + Imgproc.CHAIN_APPROX_NONE); + for(MatOfPoint contour : contours) { + Rect boundingRect = Imgproc.boundingRect(contour); + Rect sceneRoiRect = new Rect( + boundingRect.x, + boundingRect.y, + boundingRect.width + template.width() - 1, + boundingRect.height + template.height() - 1); + Mat sceneRoi = new Mat(scene, sceneRoiRect); + Imgproc.matchTemplate( + sceneRoi, + template, + new Mat(matchResult, boundingRect), + method); + } + prevMatchResult = matchResult; + } + return prevMatchResult; + } + public static Mat rotateImage(Mat img, double angleDegrees) { Point center = new Point(img.cols() / 2, img.rows() / 2); Mat rotMat = Imgproc.getRotationMatrix2D(center, -angleDegrees, 1.0); @@ -55,17 +153,19 @@ public static Mat bitmapToBinaryMat(Bitmap b, double scaleFactor) { } private class PrecomputedTileImageData { - public final Map> precomputedImages = new HashMap<>(); + public final Map>> precomputedImages = new HashMap<>(); public PrecomputedTileImageData(TileType tt, Bitmap baseImage) { // extra 0.5 factor because sample images were double size from screenshot Mat binaryMat = bitmapToBinaryMat(baseImage, globalScaleFactor * 0.5); for(double scale : tileImageScalesRange) { - Map orientationMap = new HashMap<>(); + Map> orientationMap = new HashMap<>(); Mat resizedMat = new Mat(); Imgproc.resize(binaryMat, resizedMat, new Size(), scale, scale, Imgproc.INTER_AREA); for(TileOrientation o : tt.getPossibleOrientations()) { Mat rotatedScaledImage = rotateImage(resizedMat, o.getAngle()); - orientationMap.put(o, rotatedScaledImage); + List pyramid = new ArrayList<>(); + buildPyramid(rotatedScaledImage, pyramid, PYRAMID_LEVELS); + orientationMap.put(o, pyramid); } precomputedImages.put(scale, orientationMap); } @@ -73,7 +173,7 @@ public PrecomputedTileImageData(TileType tt, Bitmap baseImage) { } public ImageProcessor(AssetManager assMan) { - for (double s = 1.0; s > 0.77; s -= 0.02) { + for (double s = 1.0; s > 0.75; s -= 0.02) { tileImageScalesRange.add(s); } for(TileType t: TileType.values()) { @@ -118,6 +218,9 @@ public GuessRecord( } public GameState getGameStateFromImage(Bitmap b) { + if (PROFILE) { + android.os.Debug.startMethodTracing(); + } // Seems opencv doesnt handle the bitmap very well when // there's aligment, so we copy it here to unalign it Bitmap unalignedBitmap = b.copy(b.getConfig(), true); @@ -176,17 +279,23 @@ public GameState getGameStateFromImage(Bitmap b) { if (DEBUG) { Debug.sendMatrix(gameImageRoi); } - double derivedScale = getTileScale(gameImageRoi); + ArrayList gameImageRoiPyramid = new ArrayList<>(); + buildPyramid(gameImageRoi, gameImageRoiPyramid, PYRAMID_LEVELS); + double derivedScale = getTileScale(gameImageRoiPyramid); Log.d(TAG, String.format("Tile scale: %s", derivedScale)); for (Map.Entry tileTypeEntry : mTileImages.entrySet()) { - Map orientationImageMap = tileTypeEntry.getValue().precomputedImages.get(derivedScale); - for (Map.Entry tileOrientationEntry : orientationImageMap.entrySet()) { - Mat tileImageToMatch = tileOrientationEntry.getValue(); + Map> orientationImageMap = tileTypeEntry.getValue().precomputedImages.get(derivedScale); + for (Map.Entry> tileOrientationEntry : orientationImageMap.entrySet()) { + List tileImageToMatchPyr = tileOrientationEntry.getValue(); + Mat tileImageToMatch = tileImageToMatchPyr.get(0); if(DEBUG) { Debug.sendMatrix(tileImageToMatch); } - Mat match = new Mat(); - Imgproc.matchTemplate(gameImageRoi, tileImageToMatch, match, Imgproc.TM_SQDIFF_NORMED); + Mat match = fastMatchTemplate( + gameImageRoiPyramid, + tileImageToMatchPyr, + Imgproc.TM_SQDIFF_NORMED, + SQDIFF_NORMED_FAST_MATCH_CALLBACK); Mat matchThreshed = new Mat(); Imgproc.threshold(match, matchThreshed, 0.3, 255, Imgproc.THRESH_BINARY_INV); Mat eightBitMatchThreshed = new Mat(); @@ -324,9 +433,10 @@ public GameState getGameStateFromImage(Bitmap b) { guess.matchedImage.height()); Mat roi = new Mat(guessesVisualized, roiRect); if (guess.type.equals(TileType.EMPTY)) { - Core.add(roi, new Scalar(127), roi); + Mat grey = new Mat(roiRect.height, roiRect.width, CvType.CV_8UC1, new Scalar(127)); + Core.addWeighted(roi, 0.8, grey, 0.8, 0, roi); } else { - Core.add(roi, guess.matchedImage, roi); + Core.addWeighted(roi, 0.8, guess.matchedImage, 0.8, 0, roi); } } @@ -396,7 +506,7 @@ public GameState getGameStateFromImage(Bitmap b) { for (int rowIndex = 0; rowIndex < rows; ++rowIndex) { TileState t = gridState[colIndex][rowIndex]; if (t.type != TileType.EMPTY) { - final Mat tileImage = mTileImages.get(t.type).precomputedImages.get(derivedScale).get(t.orientation); + final Mat tileImage = mTileImages.get(t.type).precomputedImages.get(derivedScale).get(t.orientation).get(0); final Mat resizedTileImage = new Mat(); Imgproc.resize(tileImage, resizedTileImage, new Size(gridInfo.colWidth, gridInfo.rowHeight)); Rect roiRect = new Rect( @@ -406,7 +516,7 @@ public GameState getGameStateFromImage(Bitmap b) { resizedTileImage.height()); //Log.d(TAG, roiRect.toString()); Mat roi = new Mat(debugImage, roiRect); - Core.add(roi, resizedTileImage, roi); + Core.addWeighted(roi, 0.8, resizedTileImage, 0.8, 0, roi); } } } @@ -416,6 +526,9 @@ public GameState getGameStateFromImage(Bitmap b) { return new GameState(gridInfo, gridState); } finally { unalignedBitmap.recycle(); + if (PROFILE) { + android.os.Debug.stopMethodTracing(); + } } } @@ -437,19 +550,22 @@ private static Rect getTilesBoundingRect(Mat gameImage) { return Imgproc.boundingRect(nonZeroPoints); } - private double getTileScale(Mat gameImage) { + private double getTileScale(ArrayList gameImageRoiPyramid) { // A level MUST have either Corners or End tiles in order to be topologically sound. // So we scan for these two types and see what scale image matches best in order to detect // the scale factor for images for the full sweep final TileType[] scaleSamplingTileTypes = {TileType.CORNER, TileType.END}; for (TileType type : scaleSamplingTileTypes) { Map scaleScoreMapping = new HashMap<>(); - for(Map.Entry> tileScaleEntry : mTileImages.get(type).precomputedImages.entrySet()) { + for(Map.Entry>> tileScaleEntry : mTileImages.get(type).precomputedImages.entrySet()) { double bestScore = 1.0; - for(Map.Entry tileOrientationEntry : tileScaleEntry.getValue().entrySet()) { - Mat tileImageToMatch = tileOrientationEntry.getValue(); - Mat match = new Mat(); - Imgproc.matchTemplate(gameImage, tileImageToMatch, match, Imgproc.TM_SQDIFF_NORMED); + for(Map.Entry> tileOrientationEntry : tileScaleEntry.getValue().entrySet()) { + List tileImageToMatchPyr = tileOrientationEntry.getValue(); + Mat match = fastMatchTemplate( + gameImageRoiPyramid, + tileImageToMatchPyr, + Imgproc.TM_SQDIFF_NORMED, + SQDIFF_NORMED_FAST_MATCH_CALLBACK); final Core.MinMaxLocResult minMaxLocResult = Core.minMaxLoc(match); bestScore = Math.min(bestScore, minMaxLocResult.minVal); }