diff --git a/.gitignore b/.gitignore
index 80a6566..ada93c2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -33,6 +33,7 @@ proguard/
# Log Files
*.log
+*.trace
# Dreaded DS_Store
.DS_Store
diff --git a/README.md b/README.md
index cab539a..80b6577 100644
--- a/README.md
+++ b/README.md
@@ -24,7 +24,11 @@ Make sure to pick the right architecture apk for the target device
### Launch the application:
- adb shell am instrument -w efokschaner.infinityloopsolver/efokschaner.infinityloopsolver.SolverInstrumentation
+ adb shell am instrument -w efokschaner.infinityloopsolver/.SolverInstrumentation
The application only works when launched this way as it uses UiAutomation api's that are not
available to normally launched applications.
+
+*Protip:* Use `nohup` to let the application stay running when adb is disconnected:
+ adb shell 'nohup am instrument -w efokschaner.infinityloopsolver/.SolverInstrumentation &1 >/dev/null'
+ ^C (stopping adb doesn't break the app)
\ No newline at end of file
diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml
index 4353bfc..c67f501 100644
--- a/app/src/main/AndroidManifest.xml
+++ b/app/src/main/AndroidManifest.xml
@@ -3,6 +3,7 @@
package="efokschaner.infinityloopsolver">
+
mTileImages = new HashMap<>();
private static double globalScaleFactor = 0.5;
private final ArrayList tileImageScalesRange = new ArrayList<>();
+ // cv::BuildPyramid from Imgproc
+ public static void buildPyramid(Mat src, List dst, int maxlevel, int borderType) {
+ if(BuildConfig.DEBUG) {
+ if(borderType == Core.BORDER_CONSTANT) {
+ throw new AssertionError("No support for Core.BORDER_CONSTANT");
+ }
+ }
+ dst.clear();
+ dst.add(src.clone());
+ Mat prevLevel = dst.get(0);
+ for (int i = 1; i <= maxlevel; ++i) {
+ Mat nextLevel = new Mat();
+ Imgproc.pyrDown(prevLevel, nextLevel, new Size(), borderType);
+ dst.add(nextLevel);
+ prevLevel = nextLevel;
+ }
+ }
+
+ // cv::BuildPyramid from Imgproc
+ public static void buildPyramid(Mat src, List dst, int maxlevel) {
+ buildPyramid(src, dst, maxlevel, Core.BORDER_DEFAULT);
+ }
+
+
+ // This callback should do two things.
+ // 1. Return a thresholded copy of match that masks the region to check on the next iteration
+ // 2. Reset the match Mat so that it possible to accumulate fresh matchResults into it.
+ // This may involve zeroing it out or filling it with large vals depending on your choice
+ // of matchTemplate method / thresholds etc.
+ public interface FastMatchThresholdCallback {
+ Mat call(Mat match);
+ }
+
+ public static Mat fastMatchTemplate(
+ List scenePyr,
+ List templatePyr,
+ int method,
+ FastMatchThresholdCallback cb) {
+ final int maxLevel = Math.min(scenePyr.size(), templatePyr.size()) - 1;
+ Mat prevMatchResult = new Mat();
+ Imgproc.matchTemplate(scenePyr.get(maxLevel), templatePyr.get(maxLevel), prevMatchResult, method);
+ for (int curLevel = maxLevel - 1; curLevel >= 0; --curLevel) {
+ Mat scene = scenePyr.get(curLevel);
+ Mat template = templatePyr.get(curLevel);
+ Mat prevMatchResultUp = new Mat();
+ Imgproc.pyrUp(prevMatchResult, prevMatchResultUp);
+ // prevMatchResult is conceptually an identical space to the new matchResult,
+ // but due to quantisation errors in the halving / doubling process it can be slightly
+ // different size. We'll resize it to be identical though as it allows for less
+ // defensive coding in the subsequent operations
+ Mat prevMatchResultResized = new Mat();
+ Size newMatchResultSize = new Size(
+ scene.width() - template.width() + 1,
+ scene.height() - template.height() + 1);
+ Imgproc.resize(prevMatchResult, prevMatchResultResized, newMatchResultSize);
+ Mat prevMatchResultThreshed = cb.call(prevMatchResultResized);
+ // Renaming for clarity as the callback should have reset the matrix
+ Mat matchResult = prevMatchResultResized;
+ Mat mask8u = new Mat();
+ prevMatchResultThreshed.convertTo(mask8u, CvType.CV_8U);
+ List contours = new ArrayList<>();
+ Imgproc.findContours(
+ mask8u,
+ contours,
+ new Mat(),
+ Imgproc.RETR_EXTERNAL,
+ Imgproc.CHAIN_APPROX_NONE);
+ for(MatOfPoint contour : contours) {
+ Rect boundingRect = Imgproc.boundingRect(contour);
+ Rect sceneRoiRect = new Rect(
+ boundingRect.x,
+ boundingRect.y,
+ boundingRect.width + template.width() - 1,
+ boundingRect.height + template.height() - 1);
+ Mat sceneRoi = new Mat(scene, sceneRoiRect);
+ Imgproc.matchTemplate(
+ sceneRoi,
+ template,
+ new Mat(matchResult, boundingRect),
+ method);
+ }
+ prevMatchResult = matchResult;
+ }
+ return prevMatchResult;
+ }
+
public static Mat rotateImage(Mat img, double angleDegrees) {
Point center = new Point(img.cols() / 2, img.rows() / 2);
Mat rotMat = Imgproc.getRotationMatrix2D(center, -angleDegrees, 1.0);
@@ -55,17 +153,19 @@ public static Mat bitmapToBinaryMat(Bitmap b, double scaleFactor) {
}
private class PrecomputedTileImageData {
- public final Map> precomputedImages = new HashMap<>();
+ public final Map>> precomputedImages = new HashMap<>();
public PrecomputedTileImageData(TileType tt, Bitmap baseImage) {
// extra 0.5 factor because sample images were double size from screenshot
Mat binaryMat = bitmapToBinaryMat(baseImage, globalScaleFactor * 0.5);
for(double scale : tileImageScalesRange) {
- Map orientationMap = new HashMap<>();
+ Map> orientationMap = new HashMap<>();
Mat resizedMat = new Mat();
Imgproc.resize(binaryMat, resizedMat, new Size(), scale, scale, Imgproc.INTER_AREA);
for(TileOrientation o : tt.getPossibleOrientations()) {
Mat rotatedScaledImage = rotateImage(resizedMat, o.getAngle());
- orientationMap.put(o, rotatedScaledImage);
+ List pyramid = new ArrayList<>();
+ buildPyramid(rotatedScaledImage, pyramid, PYRAMID_LEVELS);
+ orientationMap.put(o, pyramid);
}
precomputedImages.put(scale, orientationMap);
}
@@ -73,7 +173,7 @@ public PrecomputedTileImageData(TileType tt, Bitmap baseImage) {
}
public ImageProcessor(AssetManager assMan) {
- for (double s = 1.0; s > 0.77; s -= 0.02) {
+ for (double s = 1.0; s > 0.75; s -= 0.02) {
tileImageScalesRange.add(s);
}
for(TileType t: TileType.values()) {
@@ -118,6 +218,9 @@ public GuessRecord(
}
public GameState getGameStateFromImage(Bitmap b) {
+ if (PROFILE) {
+ android.os.Debug.startMethodTracing();
+ }
// Seems opencv doesnt handle the bitmap very well when
// there's aligment, so we copy it here to unalign it
Bitmap unalignedBitmap = b.copy(b.getConfig(), true);
@@ -176,17 +279,23 @@ public GameState getGameStateFromImage(Bitmap b) {
if (DEBUG) {
Debug.sendMatrix(gameImageRoi);
}
- double derivedScale = getTileScale(gameImageRoi);
+ ArrayList gameImageRoiPyramid = new ArrayList<>();
+ buildPyramid(gameImageRoi, gameImageRoiPyramid, PYRAMID_LEVELS);
+ double derivedScale = getTileScale(gameImageRoiPyramid);
Log.d(TAG, String.format("Tile scale: %s", derivedScale));
for (Map.Entry tileTypeEntry : mTileImages.entrySet()) {
- Map orientationImageMap = tileTypeEntry.getValue().precomputedImages.get(derivedScale);
- for (Map.Entry tileOrientationEntry : orientationImageMap.entrySet()) {
- Mat tileImageToMatch = tileOrientationEntry.getValue();
+ Map> orientationImageMap = tileTypeEntry.getValue().precomputedImages.get(derivedScale);
+ for (Map.Entry> tileOrientationEntry : orientationImageMap.entrySet()) {
+ List tileImageToMatchPyr = tileOrientationEntry.getValue();
+ Mat tileImageToMatch = tileImageToMatchPyr.get(0);
if(DEBUG) {
Debug.sendMatrix(tileImageToMatch);
}
- Mat match = new Mat();
- Imgproc.matchTemplate(gameImageRoi, tileImageToMatch, match, Imgproc.TM_SQDIFF_NORMED);
+ Mat match = fastMatchTemplate(
+ gameImageRoiPyramid,
+ tileImageToMatchPyr,
+ Imgproc.TM_SQDIFF_NORMED,
+ SQDIFF_NORMED_FAST_MATCH_CALLBACK);
Mat matchThreshed = new Mat();
Imgproc.threshold(match, matchThreshed, 0.3, 255, Imgproc.THRESH_BINARY_INV);
Mat eightBitMatchThreshed = new Mat();
@@ -324,9 +433,10 @@ public GameState getGameStateFromImage(Bitmap b) {
guess.matchedImage.height());
Mat roi = new Mat(guessesVisualized, roiRect);
if (guess.type.equals(TileType.EMPTY)) {
- Core.add(roi, new Scalar(127), roi);
+ Mat grey = new Mat(roiRect.height, roiRect.width, CvType.CV_8UC1, new Scalar(127));
+ Core.addWeighted(roi, 0.8, grey, 0.8, 0, roi);
} else {
- Core.add(roi, guess.matchedImage, roi);
+ Core.addWeighted(roi, 0.8, guess.matchedImage, 0.8, 0, roi);
}
}
@@ -396,7 +506,7 @@ public GameState getGameStateFromImage(Bitmap b) {
for (int rowIndex = 0; rowIndex < rows; ++rowIndex) {
TileState t = gridState[colIndex][rowIndex];
if (t.type != TileType.EMPTY) {
- final Mat tileImage = mTileImages.get(t.type).precomputedImages.get(derivedScale).get(t.orientation);
+ final Mat tileImage = mTileImages.get(t.type).precomputedImages.get(derivedScale).get(t.orientation).get(0);
final Mat resizedTileImage = new Mat();
Imgproc.resize(tileImage, resizedTileImage, new Size(gridInfo.colWidth, gridInfo.rowHeight));
Rect roiRect = new Rect(
@@ -406,7 +516,7 @@ public GameState getGameStateFromImage(Bitmap b) {
resizedTileImage.height());
//Log.d(TAG, roiRect.toString());
Mat roi = new Mat(debugImage, roiRect);
- Core.add(roi, resizedTileImage, roi);
+ Core.addWeighted(roi, 0.8, resizedTileImage, 0.8, 0, roi);
}
}
}
@@ -416,6 +526,9 @@ public GameState getGameStateFromImage(Bitmap b) {
return new GameState(gridInfo, gridState);
} finally {
unalignedBitmap.recycle();
+ if (PROFILE) {
+ android.os.Debug.stopMethodTracing();
+ }
}
}
@@ -437,19 +550,22 @@ private static Rect getTilesBoundingRect(Mat gameImage) {
return Imgproc.boundingRect(nonZeroPoints);
}
- private double getTileScale(Mat gameImage) {
+ private double getTileScale(ArrayList gameImageRoiPyramid) {
// A level MUST have either Corners or End tiles in order to be topologically sound.
// So we scan for these two types and see what scale image matches best in order to detect
// the scale factor for images for the full sweep
final TileType[] scaleSamplingTileTypes = {TileType.CORNER, TileType.END};
for (TileType type : scaleSamplingTileTypes) {
Map scaleScoreMapping = new HashMap<>();
- for(Map.Entry> tileScaleEntry : mTileImages.get(type).precomputedImages.entrySet()) {
+ for(Map.Entry>> tileScaleEntry : mTileImages.get(type).precomputedImages.entrySet()) {
double bestScore = 1.0;
- for(Map.Entry tileOrientationEntry : tileScaleEntry.getValue().entrySet()) {
- Mat tileImageToMatch = tileOrientationEntry.getValue();
- Mat match = new Mat();
- Imgproc.matchTemplate(gameImage, tileImageToMatch, match, Imgproc.TM_SQDIFF_NORMED);
+ for(Map.Entry> tileOrientationEntry : tileScaleEntry.getValue().entrySet()) {
+ List tileImageToMatchPyr = tileOrientationEntry.getValue();
+ Mat match = fastMatchTemplate(
+ gameImageRoiPyramid,
+ tileImageToMatchPyr,
+ Imgproc.TM_SQDIFF_NORMED,
+ SQDIFF_NORMED_FAST_MATCH_CALLBACK);
final Core.MinMaxLocResult minMaxLocResult = Core.minMaxLoc(match);
bestScore = Math.min(bestScore, minMaxLocResult.minVal);
}