Add androidx.benchmark.iterations argument for controlling measurement iters Test: ./gradlew benchmark:benchmark-common:cC -P android.testInstrumentationRunnerArguments.androidx.benchmark.iterations=100 -P android.testInstrumentationRunnerArguments.class=androidx.benchmark.BenchmarkStateTest Fixes: 194137879 Note: this doesn't modify warmup, which is still an arbitrary number of iterations. The initial intent is to support profiling use cases (which should use androidx.benchmark.profiling.*), or experimenting locally when providing feedback on automatic iteration determination. Change-Id: I1682ccff011f430da564cedf9113fb628b8d87b4
diff --git a/benchmark/benchmark-common/src/androidTest/java/androidx/benchmark/ArgumentInjectingApplication.kt b/benchmark/benchmark-common/src/androidTest/java/androidx/benchmark/ArgumentInjectingApplication.kt index 6f2d759..3a06fab 100644 --- a/benchmark/benchmark-common/src/androidTest/java/androidx/benchmark/ArgumentInjectingApplication.kt +++ b/benchmark/benchmark-common/src/androidTest/java/androidx/benchmark/ArgumentInjectingApplication.kt
@@ -39,6 +39,9 @@ super.onCreate() argumentSource = Bundle().apply { + // allow cli args to pass through + putAll(InstrumentationRegistry.getArguments()) + // Since these benchmark correctness tests run as part of the regular // (non-performance-test) suite, they will have debuggable=true, won't be clock-locked, // can run with low-battery or on an emulator, and code coverage enabled.
diff --git a/benchmark/benchmark-common/src/androidTest/java/androidx/benchmark/BenchmarkStateTest.kt b/benchmark/benchmark-common/src/androidTest/java/androidx/benchmark/BenchmarkStateTest.kt index 0deb870..f05047f 100644 --- a/benchmark/benchmark-common/src/androidTest/java/androidx/benchmark/BenchmarkStateTest.kt +++ b/benchmark/benchmark-common/src/androidTest/java/androidx/benchmark/BenchmarkStateTest.kt
@@ -17,7 +17,6 @@ package androidx.benchmark import android.Manifest -import android.util.Log import androidx.benchmark.BenchmarkState.Companion.ExperimentalExternalReport import androidx.test.ext.junit.runners.AndroidJUnit4 import androidx.test.filters.FlakyTest @@ -29,6 +28,7 @@ import org.junit.Assert.assertNotNull import org.junit.Assert.assertTrue import org.junit.Assert.fail +import org.junit.Assume.assumeTrue import org.junit.Rule import org.junit.Test import org.junit.runner.RunWith @@ -166,6 +166,10 @@ val expectedCount = report.warmupIterations + report.repeatIterations * expectedRepeatCount assertEquals(expectedCount, total) + if (Arguments.iterations != null) { + assertEquals(Arguments.iterations, report.repeatIterations) + } + // verify we're not in warmup mode assertTrue(report.warmupIterations > 0) assertTrue(report.repeatIterations > 1) @@ -180,16 +184,14 @@ @Test public fun iterationCheck_withAllocations() { - if (CpuInfo.locked || - IsolationActivity.sustainedPerformanceModeInUse || - Errors.isEmulator - ) { - // In any of these conditions, it's known that throttling won't happen, so it's safe - // to check for allocation count, by setting checkingForThermalThrottling = false - iterationCheck(checkingForThermalThrottling = false) - } else { - Log.d(BenchmarkState.TAG, "Warning - bypassing iterationCheck_withAllocations") - } + // In any of these conditions, it's known that throttling won't happen, so it's safe + // to check for allocation count, by setting checkingForThermalThrottling = false + assumeTrue( + CpuInfo.locked || + IsolationActivity.sustainedPerformanceModeInUse || + Errors.isEmulator + ) + iterationCheck(checkingForThermalThrottling = false) } @Test
diff --git a/benchmark/benchmark-common/src/main/java/androidx/benchmark/Arguments.kt b/benchmark/benchmark-common/src/main/java/androidx/benchmark/Arguments.kt index c14a978..04d0004 100644 --- a/benchmark/benchmark-common/src/main/java/androidx/benchmark/Arguments.kt +++ b/benchmark/benchmark-common/src/main/java/androidx/benchmark/Arguments.kt
@@ -42,6 +42,7 @@ internal val outputEnable: Boolean internal val startupMode: Boolean internal val dryRunMode: Boolean + internal val iterations: Int? internal val profiler: Profiler? internal val profilerSampleFrequency: Int internal val profilerSampleDurationSeconds: Long @@ -81,6 +82,9 @@ outputEnable = !dryRunMode && (arguments.getBenchmarkArgument("output.enable")?.toBoolean() ?: true) + iterations = + arguments.getBenchmarkArgument("iterations")?.toInt() + // Transform comma-delimited list into set of suppressed errors // E.g. "DEBUGGABLE, UNLOCKED" -> setOf("DEBUGGABLE", "UNLOCKED") suppressedErrors = arguments.getBenchmarkArgument("suppressErrors", "")
diff --git a/benchmark/benchmark-common/src/main/java/androidx/benchmark/BenchmarkState.kt b/benchmark/benchmark-common/src/main/java/androidx/benchmark/BenchmarkState.kt index b7d6f56..518fbc7 100644 --- a/benchmark/benchmark-common/src/main/java/androidx/benchmark/BenchmarkState.kt +++ b/benchmark/benchmark-common/src/main/java/androidx/benchmark/BenchmarkState.kt
@@ -440,11 +440,9 @@ } private fun computeMaxIterations(): Int { - var idealIterations = - (REPEAT_DURATION_TARGET_NS / warmupManager.estimatedIterationTimeNs).toInt() - idealIterations = idealIterations.coerceIn(MIN_TEST_ITERATIONS, MAX_TEST_ITERATIONS) - OVERRIDE_ITERATIONS?.let { idealIterations = OVERRIDE_ITERATIONS } - return idealIterations + return OVERRIDE_ITERATIONS + ?: (REPEAT_DURATION_TARGET_NS / warmupManager.estimatedIterationTimeNs).toInt() + .coerceIn(MIN_TEST_ITERATIONS, MAX_TEST_ITERATIONS) } private fun throwIfPaused() = check(!paused) { @@ -547,11 +545,13 @@ internal const val REPEAT_COUNT_ALLOCATION = 5 - private val OVERRIDE_ITERATIONS = if ( + private val OVERRIDE_ITERATIONS = when { Arguments.dryRunMode || - Arguments.startupMode || - Arguments.profiler?.requiresSingleMeasurementIteration == true - ) 1 else null + Arguments.startupMode || + Arguments.profiler?.requiresSingleMeasurementIteration == true -> 1 + Arguments.iterations != null -> Arguments.iterations + else -> null + } internal val REPEAT_DURATION_TARGET_NS = when (Arguments.profiler?.requiresExtraRuntime) { // longer measurements while profiling to ensure we have enough data
diff --git a/benchmark/benchmark-junit4/src/androidTest/java/androidx/benchmark/junit4/ArgumentInjectingApplication.kt b/benchmark/benchmark-junit4/src/androidTest/java/androidx/benchmark/junit4/ArgumentInjectingApplication.kt index c8085ea..202b117 100644 --- a/benchmark/benchmark-junit4/src/androidTest/java/androidx/benchmark/junit4/ArgumentInjectingApplication.kt +++ b/benchmark/benchmark-junit4/src/androidTest/java/androidx/benchmark/junit4/ArgumentInjectingApplication.kt
@@ -19,6 +19,7 @@ import android.app.Application import android.os.Bundle import androidx.benchmark.argumentSource +import androidx.test.platform.app.InstrumentationRegistry /** * Hack to enable overriding benchmark arguments (since we can't easily do this in CI, per apk) @@ -39,7 +40,8 @@ super.onCreate() argumentSource = Bundle().apply { - putString("androidx.benchmark.output.enable", "true") + // allow cli args to pass through + putAll(InstrumentationRegistry.getArguments()) // Since these benchmark correctness tests run as part of the regular // (non-performance-test) suite, they will have debuggable=true, won't be clock-locked,