Add support for trace processor queries

ChrisCraik · ChrisCraik · commit 869b9bb18e64 · 2021-07-27T13:27:25.000-07:00
Fixes:194808912
Test: ./gradlew benchmark:benchmark-macro:cC

Additionally, changes test previously only checking file size to also
validate trace contents. This revealed an issue that trace sections
weren't captured immediately after starting the trace, which is
currently worked around with a Thread.sleep().

Additionally, adds 'FileLinkingRule' internal test API for more easy
trace and file diagnostic issues, by enabling files produced by tests
(especially traces) to be easily reported to Studio as links.

Change-Id: I4c05e1bd0749c8d7aad7b36bff877916d2a6a9aa
diff --git a/benchmark/benchmark-macro/src/androidTest/java/androidx/benchmark/macro/FileLinkingRule.kt b/benchmark/benchmark-macro/src/androidTest/java/androidx/benchmark/macro/FileLinkingRule.kt
@@ -0,0 +1,120 @@
+/*
+ * Copyright 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package androidx.benchmark.macro
+
+import androidx.benchmark.InstrumentationResults
+import androidx.benchmark.Outputs
+import androidx.benchmark.macro.perfetto.UiState
+import androidx.benchmark.macro.perfetto.appendUiState
+import org.junit.rules.RuleChain
+import org.junit.rules.TestRule
+import org.junit.runner.Description
+import org.junit.runners.model.Statement
+import java.io.File
+
+/**
+ * Rule to enable linking files and traces to Studio UI for macrobench correctness tests.
+ *
+ * Filepaths are registered, and reported, but files are not created by this class, that should
+ * be handled by the test. Ensure you don't clean up the file - it needs to persist to be copied
+ * over by Studio.
+ */
+class FileLinkingRule : TestRule {
+    private lateinit var currentDescription: Description
+    private var summaryString = ""
+
+    private fun createReportedFilePath(
+        label: String,
+        @Suppress("SameParameterValue") extension: String,
+    ): String {
+        // remove parens / brackets, as it confuses linking
+        val methodLabel = currentDescription.toUniqueName()
+            .replace("(", "_")
+            .replace(")", "_")
+            .replace("[", "_")
+            .replace("]", "_")
+
+        val file = File(
+            Outputs.dirUsableByAppAndShell,
+            "${label}_${Outputs.dateToFileName()}.$extension"
+        )
+        val absolutePath: String = file.absolutePath
+        val relativePath = Outputs.relativePathFor(absolutePath)
+
+        summaryString += "$methodLabel [$label](file://$relativePath)\n"
+        return absolutePath
+    }
+
+    /**
+     * Map of trace abs path -> process to highlight.
+     *
+     * After trace is complete (at end of test), we write a UI state packet to it, so trace UI
+     * can highlight/select the relevant process.
+     */
+    private val traceToPackageMap = mutableMapOf<String, String>()
+
+    fun createReportedTracePath(
+        packageName: String,
+        label: String = "trace"
+    ): String {
+        val absolutePath = createReportedFilePath(label, "perfetto-trace")
+        traceToPackageMap[absolutePath] = packageName
+        return absolutePath
+    }
+
+    override fun apply(base: Statement, description: Description): Statement {
+        return RuleChain
+            .outerRule(::applyInternal)
+            .apply(base, description)
+    }
+
+    private fun applyInternal(base: Statement, description: Description) = object : Statement() {
+        override fun evaluate() {
+            require(Outputs.outputDirectory == Outputs.dirUsableByAppAndShell) {
+                "FileLinkingRule may only be used when outputDirectory == dirUsableByAppAndShell"
+            }
+
+            currentDescription = description
+            try {
+                base.evaluate()
+            } finally {
+                flush()
+            }
+        }
+    }
+
+    private fun flush() {
+        traceToPackageMap.forEach { entry ->
+            File(entry.key).apply {
+                if (exists()) {
+                    appendUiState(
+                        UiState(null, null, entry.value)
+                    )
+                }
+            }
+        }
+
+        InstrumentationResults.instrumentationReport {
+            ideSummaryRecord(
+                summaryV1 = "", // not supported
+                summaryV2 = summaryString.trim()
+            )
+        }
+    }
+
+    private fun Description.toUniqueName() = testClass.simpleName + "_" + methodName
+}
diff --git a/benchmark/benchmark-macro/src/androidTest/java/androidx/benchmark/macro/MacrobenchmarkScopeTest.kt b/benchmark/benchmark-macro/src/androidTest/java/androidx/benchmark/macro/MacrobenchmarkScopeTest.kt
@@ -44,31 +44,31 @@ public class MacrobenchmarkScopeTest {
         // since error messages from e.g. startActivityAndWait may be less clear
         try {
             val pm = InstrumentationRegistry.getInstrumentation().context.packageManager
-            pm.getApplicationInfo(TARGET_PACKAGE_NAME, 0)
+            pm.getApplicationInfo(Packages.TARGET, 0)
         } catch (notFoundException: PackageManager.NameNotFoundException) {
             throw IllegalStateException(
-                "Unable to find target $TARGET_PACKAGE_NAME, is it installed?"
+                "Unable to find target ${Packages.TARGET}, is it installed?"
             )
         }
     }
 
     @Test
     public fun killTest() {
-        val scope = MacrobenchmarkScope(TARGET_PACKAGE_NAME, launchWithClearTask = true)
+        val scope = MacrobenchmarkScope(Packages.TARGET, launchWithClearTask = true)
         scope.pressHome()
         scope.startActivityAndWait()
-        assertTrue(isProcessAlive(TARGET_PACKAGE_NAME))
+        assertTrue(isProcessAlive(Packages.TARGET))
         scope.killProcess()
-        assertFalse(isProcessAlive(TARGET_PACKAGE_NAME))
+        assertFalse(isProcessAlive(Packages.TARGET))
     }
 
     @Test
     public fun compile_speedProfile() {
-        val scope = MacrobenchmarkScope(TARGET_PACKAGE_NAME, launchWithClearTask = true)
+        val scope = MacrobenchmarkScope(Packages.TARGET, launchWithClearTask = true)
         val iterations = 1
         var executions = 0
         val compilation = CompilationMode.SpeedProfile(warmupIterations = iterations)
-        compilation.compile(TARGET_PACKAGE_NAME) {
+        compilation.compile(Packages.TARGET) {
             executions += 1
             scope.pressHome()
             scope.startActivityAndWait()
@@ -79,19 +79,19 @@ public class MacrobenchmarkScopeTest {
     @Test
     public fun compile_speed() {
         val compilation = CompilationMode.Speed
-        compilation.compile(TARGET_PACKAGE_NAME) {
+        compilation.compile(Packages.TARGET) {
             fail("Should never be called for $compilation")
         }
     }
 
     @Test
     public fun startActivityAndWait_activityNotExported() {
-        val scope = MacrobenchmarkScope(TARGET_PACKAGE_NAME, launchWithClearTask = true)
+        val scope = MacrobenchmarkScope(Packages.TARGET, launchWithClearTask = true)
         scope.pressHome()
 
         val intent = Intent()
-        intent.setPackage(TARGET_PACKAGE_NAME)
-        intent.action = "$TARGET_PACKAGE_NAME.NOT_EXPORTED_ACTIVITY"
+        intent.setPackage(Packages.TARGET)
+        intent.action = "${Packages.TARGET}.NOT_EXPORTED_ACTIVITY"
 
         val device = UiDevice.getInstance(InstrumentationRegistry.getInstrumentation())
         val prop = device.executeShellCommand("getprop service.adb.root").trim()
@@ -113,12 +113,12 @@ public class MacrobenchmarkScopeTest {
 
     @Test
     public fun startActivityAndWait_invalidActivity() {
-        val scope = MacrobenchmarkScope(TARGET_PACKAGE_NAME, launchWithClearTask = true)
+        val scope = MacrobenchmarkScope(Packages.TARGET, launchWithClearTask = true)
         scope.pressHome()
 
         val intent = Intent()
         intent.setPackage("this.is.not.a.real.package")
-        intent.action = "$TARGET_PACKAGE_NAME.NOT_EXPORTED_ACTIVITY"
+        intent.action = "${Packages.TARGET}.NOT_EXPORTED_ACTIVITY"
 
         // should throw, unable to resolve Intent
         val exceptionMessage = assertFailsWith<IllegalStateException> {
@@ -130,7 +130,10 @@ public class MacrobenchmarkScopeTest {
 
     @Test
     public fun startActivityAndWait_sameActivity() {
-        val scope = MacrobenchmarkScope(LOCAL_PACKAGE_NAME, launchWithClearTask = true)
+        val scope = MacrobenchmarkScope(
+            Packages.TEST, // self-instrumenting macrobench, so don't kill the process!
+            launchWithClearTask = true
+        )
         val device = UiDevice.getInstance(InstrumentationRegistry.getInstrumentation())
 
         // Launch first activity, and validate it is displayed
@@ -160,13 +163,4 @@ public class MacrobenchmarkScopeTest {
     private fun isProcessAlive(packageName: String): Boolean {
         return processes().any { it.contains(packageName) }
     }
-
-    public companion object {
-        // Separate target app. Use this app/package if killing/compiling target process.
-        private const val TARGET_PACKAGE_NAME =
-            "androidx.benchmark.integration.macrobenchmark.target"
-
-        // This test app. Use this app/package if not killing/compiling target.
-        private const val LOCAL_PACKAGE_NAME = "androidx.benchmark.macro.test"
-    }
 }
diff --git a/benchmark/benchmark-macro/src/androidTest/java/androidx/benchmark/macro/Packages.kt b/benchmark/benchmark-macro/src/androidTest/java/androidx/benchmark/macro/Packages.kt
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package androidx.benchmark.macro
+
+object Packages {
+    /**
+     * Separate target app.
+     *
+     * Use this app/package if it's necessary to kill/compile target process.
+     */
+    const val TARGET =
+        "androidx.benchmark.integration.macrobenchmark.target"
+
+    /**
+     * This test app - this process.
+     *
+     * Preferably use this app/package if not killing/compiling target.
+     */
+    const val TEST = "androidx.benchmark.macro.test"
+}
diff --git a/benchmark/benchmark-macro/src/androidTest/java/androidx/benchmark/macro/perfetto/PerfettoCaptureTest.kt b/benchmark/benchmark-macro/src/androidTest/java/androidx/benchmark/macro/perfetto/PerfettoCaptureTest.kt
@@ -17,7 +17,8 @@
 package androidx.benchmark.macro.perfetto
 
 import android.os.Build
-import androidx.benchmark.Outputs
+import androidx.benchmark.macro.FileLinkingRule
+import androidx.benchmark.macro.Packages
 import androidx.benchmark.macro.perfetto.PerfettoHelper.Companion.isAbiSupported
 import androidx.test.filters.LargeTest
 import androidx.test.filters.SdkSuppress
@@ -28,29 +29,31 @@ import org.junit.After
 import org.junit.Assert.assertTrue
 import org.junit.Assume.assumeTrue
 import org.junit.Before
+import org.junit.Rule
 import org.junit.Test
 import org.junit.runner.RunWith
 import org.junit.runners.Parameterized
-import java.io.File
+import kotlin.test.assertEquals
 
 @SdkSuppress(minSdkVersion = 29) // Lower to 21 after fixing trace config.
 @RunWith(Parameterized::class)
-public class PerfettoCaptureTest(private val unbundled: Boolean) {
-    private val traceFile = File(Outputs.dirUsableByAppAndShell, "PerfettoCaptureTest.trace")
-    private val traceFilePath = traceFile.absolutePath
+class PerfettoCaptureTest(private val unbundled: Boolean) {
+    @get:Rule
+    val linkRule = FileLinkingRule()
 
     @Before
     @After
-    public fun cleanup() {
+    fun cleanup() {
         PerfettoCapture(unbundled).cancel()
-        traceFile.delete()
     }
 
     @LargeTest
     @Test
-    public fun traceAndCheckFileSize() {
+    fun captureAndValidateTrace() {
         // Change the check to API >=21, once we have the correct Perfetto config.
         assumeTrue(Build.VERSION.SDK_INT >= 29 && isAbiSupported())
+
+        val traceFilePath = linkRule.createReportedTracePath(Packages.TEST)
         val perfettoCapture = PerfettoCapture(unbundled)
 
         verifyTraceEnable(false)
@@ -59,27 +62,40 @@ public class PerfettoCaptureTest(private val unbundled: Boolean) {
 
         verifyTraceEnable(true)
 
-        trace("PerfettoCaptureTest") {
+        // TODO: figure out why this sleep (200ms+) is needed - possibly related to b/194105203
+        Thread.sleep(500)
+
+        trace(TRACE_SECTION_LABEL) {
             // Tracing non-trivial duration for manual debugging/verification
             Thread.sleep(20)
         }
 
         perfettoCapture.stop(traceFilePath)
 
-        val length = traceFile.length()
-        assertTrue("Expect > 10KiB file, was $length bytes", length > 10 * 1024)
+        val matchingSlices = PerfettoTraceProcessor.querySlices(
+            absoluteTracePath = traceFilePath,
+            TRACE_SECTION_LABEL
+        )
+
+        assertEquals(1, matchingSlices.size)
+        matchingSlices.first().apply {
+            assertEquals(TRACE_SECTION_LABEL, name)
+            assertTrue(dur > 15_000_000) // should be at least 15ms
+        }
     }
 
-    public companion object {
+    companion object {
+        const val TRACE_SECTION_LABEL = "PerfettoCaptureTest"
+
         @Parameterized.Parameters(name = "unbundled={0}")
         @JvmStatic
-        public fun parameters(): Array<Any> {
+        fun parameters(): Array<Any> {
             return arrayOf(true, false)
         }
     }
 }
 
-public fun verifyTraceEnable(enabled: Boolean) {
+fun verifyTraceEnable(enabled: Boolean) {
     // We poll here, since we may need to wait for enable flags to propagate to apps
     verifyWithPolling(
         "Timeout waiting for Trace.isEnabled == $enabled",
diff --git a/benchmark/benchmark-macro/src/main/java/androidx/benchmark/macro/perfetto/PerfettoTraceProcessor.kt b/benchmark/benchmark-macro/src/main/java/androidx/benchmark/macro/perfetto/PerfettoTraceProcessor.kt
