Adapt Android test runner and rework test lists

The dEQP test runner will now skip Vulkan tests that the device does
not claim to support.  This is detected by comparing the date
associated with each Vulkan case list file with the date provided by
the android.software.vulkan.deqp.level feature flag version, and
executing only those Vulkan tests associated with the date and
earlier.

Android test list generation has been adapted to generate test lists
for dates 2019-03-01 and 2020-03-01, corresponding to Android versions
10 and 11.

Change-Id: I06fecb5804fa91d85a7e6083331bae7f7ccd42ec

android/cts/AndroidTest.xml

@@ -183,11 +183,16 @@
 	<test class="com.drawelements.deqp.runner.DeqpTestRunner">
 		<option name="deqp-package" value="dEQP-VK"/>
 		<option name="deqp-caselist-file" value="vk-master.txt"/>
+		<option name="runtime-hint" value="2h39m"/>
+	</test>
+	<test class="com.drawelements.deqp.runner.DeqpTestRunner">
+		<option name="deqp-package" value="dEQP-VK"/>
+		<option name="deqp-caselist-file" value="vk-master-2019-03-01.txt"/>
 		<option name="runtime-hint" value="2h29m"/>
 	</test>
 	<test class="com.drawelements.deqp.runner.DeqpTestRunner">
 		<option name="deqp-package" value="dEQP-VK"/>
-		<option name="deqp-caselist-file" value="vk-master-risky.txt"/>
+		<option name="deqp-caselist-file" value="vk-master-2020-03-01.txt"/>
 		<option name="runtime-hint" value="10m"/>
 	</test>
 </configuration>

android/cts/master/mustpass.xml

@@ -47,6 +47,7 @@
 	</TestPackage>
 	<TestPackage name="dEQP-VK">
 		<Configuration caseListFile="vk-master.txt" commandLine="--deqp-watchdog=enable" name="master"/>
-		<Configuration caseListFile="vk-master-risky.txt" commandLine="--deqp-watchdog=enable" name="master-risky"/>
+		<Configuration caseListFile="vk-master-2019-03-01.txt" commandLine="--deqp-watchdog=enable" name="master-2019-03-01"/>
+		<Configuration caseListFile="vk-master-2020-03-01.txt" commandLine="--deqp-watchdog=enable" name="master-2020-03-01"/>
 	</TestPackage>
 </Mustpass>

android/cts/runner/src/com/drawelements/deqp/runner/DeqpTestRunner.java

@@ -65,8 +65,10 @@
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 /**
@@ -88,6 +90,7 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest,
     public static final String FEATURE_LANDSCAPE = "android.hardware.screen.landscape";
     public static final String FEATURE_PORTRAIT = "android.hardware.screen.portrait";
     public static final String FEATURE_VULKAN_LEVEL = "android.hardware.vulkan.level";
+    public static final String FEATURE_VULKAN_DEQP_LEVEL = "android.software.vulkan.deqp.level";
 
     private static final int TESTCASE_BATCH_LIMIT = 1000;
     private static final int UNRESPONSIVE_CMD_TIMEOUT_MS = 10 * 60 * 1000; // 10min
@@ -149,7 +152,7 @@ public class DeqpTestRunner implements IBuildReceiver, IDeviceTest,
     private CompatibilityBuildHelper mBuildHelper;
     private boolean mLogData = false;
     private ITestDevice mDevice;
-    private Set<String> mDeviceFeatures;
+    private Map<String, Optional<Integer>> mDeviceFeatures;
     private Map<String, Boolean> mConfigQuerySupportCache = new HashMap<>();
     private IRunUtil mRunUtil = RunUtil.getDefault();
     // When set will override the mCaselistFile for testing purposes.
@@ -1299,14 +1302,14 @@ private boolean isSupportedRunConfiguration(BatchRunConfiguration runConfig)
             throws DeviceNotAvailableException, CapabilityQueryFailureException {
         // orientation support
         if (!BatchRunConfiguration.ROTATION_UNSPECIFIED.equals(runConfig.getRotation())) {
-            final Set<String> features = getDeviceFeatures(mDevice);
+            final Map<String, Optional<Integer>> features = getDeviceFeatures(mDevice);
 
             if (isPortraitClassRotation(runConfig.getRotation()) &&
-                    !features.contains(FEATURE_PORTRAIT)) {
+                    !features.containsKey(FEATURE_PORTRAIT)) {
                 return false;
             }
             if (isLandscapeClassRotation(runConfig.getRotation()) &&
-                    !features.contains(FEATURE_LANDSCAPE)) {
+                    !features.containsKey(FEATURE_LANDSCAPE)) {
                 return false;
             }
         }
@@ -1622,7 +1625,8 @@ private void fakePassTests(ITestInvocationListener listener) {
             listener.testEnded(test, emptyMap);
         }
         // Log only once all the skipped tests
-        CLog.d("Opengl ES version not supported. Skipping tests '%s'", mRemainingTests);
+        CLog.d("Skipping tests '%s', either because they are not supported by the device or "
+            + "because tests are simply being collected", mRemainingTests);
         mRemainingTests.clear();
     }
 
@@ -1631,9 +1635,9 @@ private void fakePassTests(ITestInvocationListener listener) {
      */
     private boolean isSupportedVulkan ()
             throws DeviceNotAvailableException, CapabilityQueryFailureException {
-        final Set<String> features = getDeviceFeatures(mDevice);
+        final Map<String, Optional<Integer>> features = getDeviceFeatures(mDevice);
 
-        for (String feature : features) {
+        for (String feature : features.keySet()) {
             if (feature.startsWith(FEATURE_VULKAN_LEVEL)) {
                 return true;
             }
@@ -1642,6 +1646,54 @@ private boolean isSupportedVulkan ()
         return false;
     }
 
+    /**
+     * Check whether the device's claimed Vulkan dEQP level is high enough that it should pass the
+     * tests in the caselist.
+     *
+     * Precondition: the package must be a Vulkan package.
+     */
+    private boolean claimedVuklanDeqpLevelIsRecentEnough()
+        throws CapabilityQueryFailureException, DeviceNotAvailableException {
+
+        if (!isVulkanPackage()) {
+            throw new AssertionError("Claims about Vulkan dEQP support should only be checked for "
+                + "Vulkan packages");
+        }
+        final Map<String, Optional<Integer>> features = getDeviceFeatures(mDevice);
+        for (String feature : features.keySet()) {
+            if (feature.startsWith(FEATURE_VULKAN_DEQP_LEVEL)) {
+                final Optional<Integer> claimedVulkanDeqpLevel = features.get(feature);
+                if (!claimedVulkanDeqpLevel.isPresent()) {
+                    throw new IllegalStateException("Feature " + FEATURE_VULKAN_DEQP_LEVEL
+                        + " has no associated version");
+                }
+                // A Vulkan case list filename has the form 'vk-master-YYYY-MM-DD.txt'
+                final Pattern caseListFilenamePattern = Pattern
+                    .compile("vk-master-(\\d\\d\\d\\d)-(\\d\\d)-(\\d\\d)\\.txt");
+                final Matcher matcher = caseListFilenamePattern.matcher(mCaselistFile);
+                if (matcher.find()) {
+                    final int year = Integer.parseInt(matcher.group(1));
+                    final int month = Integer.parseInt(matcher.group(2));
+                    final int day = Integer.parseInt(matcher.group(3));
+                    CLog.d("Case list date is %s-%s-%s", String.format("04d", year),
+                        String.format("02d", month), String.format("02d", day));
+                    // As per the documentation for FEATURE_VULKAN_DEQP_LEVEL in
+                    // android.content.pm.PackageManager, a year is encoded as an integer by
+                    // devoting bits 31-16 to year, 15-8 to month and 7-0 to day
+                    final int minimumLevel = (year << 16) + (month << 8) + day;
+                    CLog.d("Minimum level required to run this case list is %d", minimumLevel);
+                    CLog.d("Claimed level for this device is %d", claimedVulkanDeqpLevel.get());
+                    return claimedVulkanDeqpLevel.get() >= minimumLevel;
+                } else {
+                    throw new IllegalStateException("Case list filename " + mCaselistFile
+                        + " is malformed");
+                }
+            }
+        }
+        throw new IllegalStateException("Device supports Vulkan, so must have feature "
+            + FEATURE_VULKAN_DEQP_LEVEL);
+    }
+
     /**
      * Check if device supports OpenGL ES version.
      */
@@ -1723,9 +1775,9 @@ private boolean queryIsSupportedConfigCommandLine(String deqpCommandLine)
     }
 
     /**
-     * Return feature set supported by the device
+     * Return feature set supported by the device, mapping integer-valued features to their values
      */
-    private Set<String> getDeviceFeatures(ITestDevice device)
+    private Map<String, Optional<Integer>> getDeviceFeatures(ITestDevice device)
             throws DeviceNotAvailableException, CapabilityQueryFailureException {
         if (mDeviceFeatures == null) {
             mDeviceFeatures = queryDeviceFeatures(device);
@@ -1736,23 +1788,38 @@ private Set<String> getDeviceFeatures(ITestDevice device)
     /**
      * Query feature set supported by the device
      */
-    private static Set<String> queryDeviceFeatures(ITestDevice device)
+    private static Map<String, Optional<Integer>> queryDeviceFeatures(ITestDevice device)
             throws DeviceNotAvailableException, CapabilityQueryFailureException {
         // NOTE: Almost identical code in BaseDevicePolicyTest#hasDeviceFeatures
         // TODO: Move this logic to ITestDevice.
         String command = "pm list features";
         String commandOutput = device.executeShellCommand(command);
 
         // Extract the id of the new user.
-        HashSet<String> availableFeatures = new HashSet<>();
+        Map<String, Optional<Integer>> availableFeatures = new HashMap<>();
         for (String feature: commandOutput.split("\\s+")) {
-            // Each line in the output of the command has the format "feature:{FEATURE_VALUE}".
-            String[] tokens = feature.split(":");
+            // Each line in the output of the command has the format "feature:{FEATURE_NAME}",
+            // optionally followed by "={FEATURE_VERSION}".
+            String[] tokens = feature.split(":|=");
             if (tokens.length < 2 || !"feature".equals(tokens[0])) {
-                CLog.e("Failed parse features. Unexpect format on line \"%s\"", tokens[0]);
+                CLog.e("Failed parse features. Unexpect format on line \"%s\"", feature);
                 throw new CapabilityQueryFailureException();
             }
-            availableFeatures.add(tokens[1]);
+            final String featureName = tokens[1];
+            Optional<Integer> featureValue = Optional.empty();
+            if (tokens.length > 2) {
+                try {
+                    // Integer.decode, rather than Integer.parseInt, is used here since some
+                    // feature versions may be presented in decimal and others in hexadecimal.
+                    featureValue = Optional.of(Integer.decode(tokens[2]));
+                } catch (NumberFormatException numberFormatException) {
+                    CLog.e("Failed parse features. Feature value \"%s\" was not an integer on "
+                        + "line \"%s\"", tokens[2], feature);
+                    throw new CapabilityQueryFailureException();
+                }
+
+            }
+            availableFeatures.put(featureName, featureValue);
         }
         return availableFeatures;
     }
@@ -2032,10 +2099,13 @@ public void run(ITestInvocationListener listener) throws DeviceNotAvailableExcep
             final boolean isSupportedApi = (isOpenGlEsPackage() && isSupportedGles())
                                             || (isVulkanPackage() && isSupportedVulkan())
                                             || (!isOpenGlEsPackage() && !isVulkanPackage());
-
-            if (!isSupportedApi || mCollectTestsOnly) {
-                // Pass all tests if OpenGL ES version is not supported or we are collecting
-                // the names of the tests only
+            final boolean deviceClaimsToSupportTests = !isVulkanPackage()
+                || claimedVuklanDeqpLevelIsRecentEnough();
+            if (!isSupportedApi || !deviceClaimsToSupportTests || mCollectTestsOnly) {
+                // Pass all tests trivially if:
+                // - the OpenGL ES or Vulkan is not supported, or
+                // - the device's feature flags do not claim to pass the tests, or
+                // - we are collecting the names of the tests only
                 fakePassTests(listener);
             } else if (!mRemainingTests.isEmpty()) {
                 mInstanceListerner.setSink(listener);

scripts/build_android_mustpass.py

@@ -243,9 +243,12 @@
 MASTER_VULKAN_PKG				= Package(module = VULKAN_MODULE, configurations = [
 		Configuration(name			= "master",
 					  filters		= MASTER_VULKAN_FILTERS,
+					  runtime		= "2h39m"),
+		Configuration(name			= "master-2019-03-01",
+					  filters		= [include("vk-master-2019-03-01.txt")],
 					  runtime		= "2h29m"),
-		Configuration(name			= "master-risky",
-					  filters		= [include("vk-temp-excluded.txt")],
+		Configuration(name			= "master-2020-03-01",
+					  filters		= MASTER_VULKAN_FILTERS + [exclude("vk-master-2019-03-01.txt")],
 					  runtime		= "10m"),
 	])