From 869b9bb18e647fbaeb4b92fd485c959eb193c8fb Mon Sep 17 00:00:00 2001 From: Chris Craik Date: Mon, 26 Jul 2021 21:04:08 -0700 Subject: [PATCH] Add support for trace processor queries Fixes:194808912 Test: ./gradlew benchmark:benchmark-macro:cC Additionally, changes test previously only checking file size to also validate trace contents. This revealed an issue that trace sections weren't captured immediately after starting the trace, which is currently worked around with a Thread.sleep(). Additionally, adds 'FileLinkingRule' internal test API for more easy trace and file diagnostic issues, by enabling files produced by tests (especially traces) to be easily reported to Studio as links. Change-Id: I4c05e1bd0749c8d7aad7b36bff877916d2a6a9aa --- .../benchmark/macro/FileLinkingRule.kt | 120 ++++++++++++++++++ .../macro/MacrobenchmarkScopeTest.kt | 40 +++--- .../java/androidx/benchmark/macro/Packages.kt | 34 +++++ .../macro/perfetto/PerfettoCaptureTest.kt | 44 +++++-- .../macro/perfetto/PerfettoTraceProcessor.kt | 81 +++++++++++- 5 files changed, 281 insertions(+), 38 deletions(-) create mode 100644 benchmark/benchmark-macro/src/androidTest/java/androidx/benchmark/macro/FileLinkingRule.kt create mode 100644 benchmark/benchmark-macro/src/androidTest/java/androidx/benchmark/macro/Packages.kt diff --git a/benchmark/benchmark-macro/src/androidTest/java/androidx/benchmark/macro/FileLinkingRule.kt b/benchmark/benchmark-macro/src/androidTest/java/androidx/benchmark/macro/FileLinkingRule.kt new file mode 100644 index 0000000000000..072e1d0f884cf --- /dev/null +++ b/benchmark/benchmark-macro/src/androidTest/java/androidx/benchmark/macro/FileLinkingRule.kt @@ -0,0 +1,120 @@ +/* + * Copyright 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.benchmark.macro + +import androidx.benchmark.InstrumentationResults +import androidx.benchmark.Outputs +import androidx.benchmark.macro.perfetto.UiState +import androidx.benchmark.macro.perfetto.appendUiState +import org.junit.rules.RuleChain +import org.junit.rules.TestRule +import org.junit.runner.Description +import org.junit.runners.model.Statement +import java.io.File + +/** + * Rule to enable linking files and traces to Studio UI for macrobench correctness tests. + * + * Filepaths are registered, and reported, but files are not created by this class, that should + * be handled by the test. Ensure you don't clean up the file - it needs to persist to be copied + * over by Studio. + */ +class FileLinkingRule : TestRule { + private lateinit var currentDescription: Description + private var summaryString = "" + + private fun createReportedFilePath( + label: String, + @Suppress("SameParameterValue") extension: String, + ): String { + // remove parens / brackets, as it confuses linking + val methodLabel = currentDescription.toUniqueName() + .replace("(", "_") + .replace(")", "_") + .replace("[", "_") + .replace("]", "_") + + val file = File( + Outputs.dirUsableByAppAndShell, + "${label}_${Outputs.dateToFileName()}.$extension" + ) + val absolutePath: String = file.absolutePath + val relativePath = Outputs.relativePathFor(absolutePath) + + summaryString += "$methodLabel [$label](file://$relativePath)\n" + return absolutePath + } + + /** + * Map of trace abs path -> process to highlight. + * + * After trace is complete (at end of test), we write a UI state packet to it, so trace UI + * can highlight/select the relevant process. + */ + private val traceToPackageMap = mutableMapOf() + + fun createReportedTracePath( + packageName: String, + label: String = "trace" + ): String { + val absolutePath = createReportedFilePath(label, "perfetto-trace") + traceToPackageMap[absolutePath] = packageName + return absolutePath + } + + override fun apply(base: Statement, description: Description): Statement { + return RuleChain + .outerRule(::applyInternal) + .apply(base, description) + } + + private fun applyInternal(base: Statement, description: Description) = object : Statement() { + override fun evaluate() { + require(Outputs.outputDirectory == Outputs.dirUsableByAppAndShell) { + "FileLinkingRule may only be used when outputDirectory == dirUsableByAppAndShell" + } + + currentDescription = description + try { + base.evaluate() + } finally { + flush() + } + } + } + + private fun flush() { + traceToPackageMap.forEach { entry -> + File(entry.key).apply { + if (exists()) { + appendUiState( + UiState(null, null, entry.value) + ) + } + } + } + + InstrumentationResults.instrumentationReport { + ideSummaryRecord( + summaryV1 = "", // not supported + summaryV2 = summaryString.trim() + ) + } + } + + private fun Description.toUniqueName() = testClass.simpleName + "_" + methodName +} diff --git a/benchmark/benchmark-macro/src/androidTest/java/androidx/benchmark/macro/MacrobenchmarkScopeTest.kt b/benchmark/benchmark-macro/src/androidTest/java/androidx/benchmark/macro/MacrobenchmarkScopeTest.kt index b2843e045b71c..a5a216d27cb05 100644 --- a/benchmark/benchmark-macro/src/androidTest/java/androidx/benchmark/macro/MacrobenchmarkScopeTest.kt +++ b/benchmark/benchmark-macro/src/androidTest/java/androidx/benchmark/macro/MacrobenchmarkScopeTest.kt @@ -44,31 +44,31 @@ public class MacrobenchmarkScopeTest { // since error messages from e.g. startActivityAndWait may be less clear try { val pm = InstrumentationRegistry.getInstrumentation().context.packageManager - pm.getApplicationInfo(TARGET_PACKAGE_NAME, 0) + pm.getApplicationInfo(Packages.TARGET, 0) } catch (notFoundException: PackageManager.NameNotFoundException) { throw IllegalStateException( - "Unable to find target $TARGET_PACKAGE_NAME, is it installed?" + "Unable to find target ${Packages.TARGET}, is it installed?" ) } } @Test public fun killTest() { - val scope = MacrobenchmarkScope(TARGET_PACKAGE_NAME, launchWithClearTask = true) + val scope = MacrobenchmarkScope(Packages.TARGET, launchWithClearTask = true) scope.pressHome() scope.startActivityAndWait() - assertTrue(isProcessAlive(TARGET_PACKAGE_NAME)) + assertTrue(isProcessAlive(Packages.TARGET)) scope.killProcess() - assertFalse(isProcessAlive(TARGET_PACKAGE_NAME)) + assertFalse(isProcessAlive(Packages.TARGET)) } @Test public fun compile_speedProfile() { - val scope = MacrobenchmarkScope(TARGET_PACKAGE_NAME, launchWithClearTask = true) + val scope = MacrobenchmarkScope(Packages.TARGET, launchWithClearTask = true) val iterations = 1 var executions = 0 val compilation = CompilationMode.SpeedProfile(warmupIterations = iterations) - compilation.compile(TARGET_PACKAGE_NAME) { + compilation.compile(Packages.TARGET) { executions += 1 scope.pressHome() scope.startActivityAndWait() @@ -79,19 +79,19 @@ public class MacrobenchmarkScopeTest { @Test public fun compile_speed() { val compilation = CompilationMode.Speed - compilation.compile(TARGET_PACKAGE_NAME) { + compilation.compile(Packages.TARGET) { fail("Should never be called for $compilation") } } @Test public fun startActivityAndWait_activityNotExported() { - val scope = MacrobenchmarkScope(TARGET_PACKAGE_NAME, launchWithClearTask = true) + val scope = MacrobenchmarkScope(Packages.TARGET, launchWithClearTask = true) scope.pressHome() val intent = Intent() - intent.setPackage(TARGET_PACKAGE_NAME) - intent.action = "$TARGET_PACKAGE_NAME.NOT_EXPORTED_ACTIVITY" + intent.setPackage(Packages.TARGET) + intent.action = "${Packages.TARGET}.NOT_EXPORTED_ACTIVITY" val device = UiDevice.getInstance(InstrumentationRegistry.getInstrumentation()) val prop = device.executeShellCommand("getprop service.adb.root").trim() @@ -113,12 +113,12 @@ public class MacrobenchmarkScopeTest { @Test public fun startActivityAndWait_invalidActivity() { - val scope = MacrobenchmarkScope(TARGET_PACKAGE_NAME, launchWithClearTask = true) + val scope = MacrobenchmarkScope(Packages.TARGET, launchWithClearTask = true) scope.pressHome() val intent = Intent() intent.setPackage("this.is.not.a.real.package") - intent.action = "$TARGET_PACKAGE_NAME.NOT_EXPORTED_ACTIVITY" + intent.action = "${Packages.TARGET}.NOT_EXPORTED_ACTIVITY" // should throw, unable to resolve Intent val exceptionMessage = assertFailsWith { @@ -130,7 +130,10 @@ public class MacrobenchmarkScopeTest { @Test public fun startActivityAndWait_sameActivity() { - val scope = MacrobenchmarkScope(LOCAL_PACKAGE_NAME, launchWithClearTask = true) + val scope = MacrobenchmarkScope( + Packages.TEST, // self-instrumenting macrobench, so don't kill the process! + launchWithClearTask = true + ) val device = UiDevice.getInstance(InstrumentationRegistry.getInstrumentation()) // Launch first activity, and validate it is displayed @@ -160,13 +163,4 @@ public class MacrobenchmarkScopeTest { private fun isProcessAlive(packageName: String): Boolean { return processes().any { it.contains(packageName) } } - - public companion object { - // Separate target app. Use this app/package if killing/compiling target process. - private const val TARGET_PACKAGE_NAME = - "androidx.benchmark.integration.macrobenchmark.target" - - // This test app. Use this app/package if not killing/compiling target. - private const val LOCAL_PACKAGE_NAME = "androidx.benchmark.macro.test" - } } diff --git a/benchmark/benchmark-macro/src/androidTest/java/androidx/benchmark/macro/Packages.kt b/benchmark/benchmark-macro/src/androidTest/java/androidx/benchmark/macro/Packages.kt new file mode 100644 index 0000000000000..7cc468ebd1bcb --- /dev/null +++ b/benchmark/benchmark-macro/src/androidTest/java/androidx/benchmark/macro/Packages.kt @@ -0,0 +1,34 @@ +/* + * Copyright 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.benchmark.macro + +object Packages { + /** + * Separate target app. + * + * Use this app/package if it's necessary to kill/compile target process. + */ + const val TARGET = + "androidx.benchmark.integration.macrobenchmark.target" + + /** + * This test app - this process. + * + * Preferably use this app/package if not killing/compiling target. + */ + const val TEST = "androidx.benchmark.macro.test" +} \ No newline at end of file diff --git a/benchmark/benchmark-macro/src/androidTest/java/androidx/benchmark/macro/perfetto/PerfettoCaptureTest.kt b/benchmark/benchmark-macro/src/androidTest/java/androidx/benchmark/macro/perfetto/PerfettoCaptureTest.kt index b8cbc9f854956..fccb981404508 100644 --- a/benchmark/benchmark-macro/src/androidTest/java/androidx/benchmark/macro/perfetto/PerfettoCaptureTest.kt +++ b/benchmark/benchmark-macro/src/androidTest/java/androidx/benchmark/macro/perfetto/PerfettoCaptureTest.kt @@ -17,7 +17,8 @@ package androidx.benchmark.macro.perfetto import android.os.Build -import androidx.benchmark.Outputs +import androidx.benchmark.macro.FileLinkingRule +import androidx.benchmark.macro.Packages import androidx.benchmark.macro.perfetto.PerfettoHelper.Companion.isAbiSupported import androidx.test.filters.LargeTest import androidx.test.filters.SdkSuppress @@ -28,29 +29,31 @@ import org.junit.After import org.junit.Assert.assertTrue import org.junit.Assume.assumeTrue import org.junit.Before +import org.junit.Rule import org.junit.Test import org.junit.runner.RunWith import org.junit.runners.Parameterized -import java.io.File +import kotlin.test.assertEquals @SdkSuppress(minSdkVersion = 29) // Lower to 21 after fixing trace config. @RunWith(Parameterized::class) -public class PerfettoCaptureTest(private val unbundled: Boolean) { - private val traceFile = File(Outputs.dirUsableByAppAndShell, "PerfettoCaptureTest.trace") - private val traceFilePath = traceFile.absolutePath +class PerfettoCaptureTest(private val unbundled: Boolean) { + @get:Rule + val linkRule = FileLinkingRule() @Before @After - public fun cleanup() { + fun cleanup() { PerfettoCapture(unbundled).cancel() - traceFile.delete() } @LargeTest @Test - public fun traceAndCheckFileSize() { + fun captureAndValidateTrace() { // Change the check to API >=21, once we have the correct Perfetto config. assumeTrue(Build.VERSION.SDK_INT >= 29 && isAbiSupported()) + + val traceFilePath = linkRule.createReportedTracePath(Packages.TEST) val perfettoCapture = PerfettoCapture(unbundled) verifyTraceEnable(false) @@ -59,27 +62,40 @@ public class PerfettoCaptureTest(private val unbundled: Boolean) { verifyTraceEnable(true) - trace("PerfettoCaptureTest") { + // TODO: figure out why this sleep (200ms+) is needed - possibly related to b/194105203 + Thread.sleep(500) + + trace(TRACE_SECTION_LABEL) { // Tracing non-trivial duration for manual debugging/verification Thread.sleep(20) } perfettoCapture.stop(traceFilePath) - val length = traceFile.length() - assertTrue("Expect > 10KiB file, was $length bytes", length > 10 * 1024) + val matchingSlices = PerfettoTraceProcessor.querySlices( + absoluteTracePath = traceFilePath, + TRACE_SECTION_LABEL + ) + + assertEquals(1, matchingSlices.size) + matchingSlices.first().apply { + assertEquals(TRACE_SECTION_LABEL, name) + assertTrue(dur > 15_000_000) // should be at least 15ms + } } - public companion object { + companion object { + const val TRACE_SECTION_LABEL = "PerfettoCaptureTest" + @Parameterized.Parameters(name = "unbundled={0}") @JvmStatic - public fun parameters(): Array { + fun parameters(): Array { return arrayOf(true, false) } } } -public fun verifyTraceEnable(enabled: Boolean) { +fun verifyTraceEnable(enabled: Boolean) { // We poll here, since we may need to wait for enable flags to propagate to apps verifyWithPolling( "Timeout waiting for Trace.isEnabled == $enabled", diff --git a/benchmark/benchmark-macro/src/main/java/androidx/benchmark/macro/perfetto/PerfettoTraceProcessor.kt b/benchmark/benchmark-macro/src/main/java/androidx/benchmark/macro/perfetto/PerfettoTraceProcessor.kt index 8fdab8f0486eb..2646dad2124f8 100644 --- a/benchmark/benchmark-macro/src/main/java/androidx/benchmark/macro/perfetto/PerfettoTraceProcessor.kt +++ b/benchmark/benchmark-macro/src/main/java/androidx/benchmark/macro/perfetto/PerfettoTraceProcessor.kt @@ -18,6 +18,7 @@ package androidx.benchmark.macro.perfetto import android.util.Log import androidx.annotation.RequiresApi +import androidx.benchmark.Outputs import androidx.benchmark.macro.device import androidx.benchmark.macro.userspaceTrace import androidx.test.platform.app.InstrumentationRegistry @@ -42,10 +43,14 @@ internal object PerfettoTraceProcessor { PerfettoHelper.createExecutable("trace_processor_shell") } - fun getJsonMetrics(absoluteTracePath: String, metric: String): String { + private fun validateTracePath(absoluteTracePath: String) { require(!absoluteTracePath.contains(" ")) { "Trace path must not contain spaces: $absoluteTracePath" } + } + + fun getJsonMetrics(absoluteTracePath: String, metric: String): String { + validateTracePath(absoluteTracePath) require(!metric.contains(" ")) { "Metric must not contain spaces: $metric" } @@ -69,4 +74,78 @@ internal object PerfettoTraceProcessor { } return json } + + data class Slice( + val name: String, + val ts: Long, + val dur: Long + ) + + private fun String.unquote(): String { + require(this.first() == '"' && this.last() == '"') + return this.substring(1, length - 1) + } + + /** + * Query a trace for a list of slices - name, timestamp, and duration. + */ + fun querySlices( + absoluteTracePath: String, + vararg sliceNames: String + ): List { + val whereClause = sliceNames + .joinToString(separator = " AND ") { + "slice.name = '$it'" + } + + val queryResult = rawQuery( + absoluteTracePath = absoluteTracePath, + query = """ + SELECT slice.name,ts,dur + FROM slice + JOIN thread_track ON thread_track.id = slice.track_id + WHERE $whereClause + """.trimMargin() + ) + val resultLines = queryResult.split("\n") + + if (resultLines.first() != "\"name\",\"ts\",\"dur\"") { + throw IllegalStateException("query failed!") + } + + // results are in CSV with a header row, and strings wrapped with quotes + return resultLines + .filter { it.isNotBlank() } // drop blank lines + .drop(1) // drop the header row + .map { + val columns = it.split(",") + Slice( + name = columns[0].unquote(), + ts = columns[1].toLong(), + dur = columns[2].toLong() + ) + } + } + + private fun rawQuery( + absoluteTracePath: String, + query: String + ): String { + validateTracePath(absoluteTracePath) + + val queryFile = File(Outputs.dirUsableByAppAndShell, "trace_processor_query.sql") + try { + queryFile.writeText(query) + + val instrumentation = InstrumentationRegistry.getInstrumentation() + val device = instrumentation.device() + + val command = "$shellPath --query-file ${queryFile.absolutePath} $absoluteTracePath" + return userspaceTrace("trace_processor_shell") { + device.executeShellCommand(command) + } + } finally { + queryFile.delete() + } + } }