Skip to content

Commit

Permalink
[Reproduce tool] Add Android emulator support. (google#1164)
Browse files Browse the repository at this point in the history
* [Reproduce tool] Add Android emulator support.

* Fix lint error

* Address review comments

* Update comment and variable names for automatic Android deps install
  • Loading branch information
mbarbella-chromium authored Nov 4, 2019
1 parent 1d50627 commit fed349e
Show file tree
Hide file tree
Showing 7 changed files with 177 additions and 37 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
/bot/*
/coverage
/deployment
/local/bin/android-sdk/*
/local/storage
/paramiko.log
/src/appengine/app.yaml
Expand Down
5 changes: 5 additions & 0 deletions butler.py
Original file line number Diff line number Diff line change
Expand Up @@ -298,6 +298,11 @@ def main():
'--verbose',
action='store_true',
help='Print additional log messages while running.')
parser_reproduce.add_argument(
'-e',
'--emulator',
action='store_true',
help='Run and attempt to reproduce a crash using the Android emulator.')

args = parser.parse_args()

Expand Down
28 changes: 27 additions & 1 deletion local/install_deps_linux.bash
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,12 @@
# Process command line arguments.
while [ "$1" != "" ]; do
case $1 in
--only-reproduce) only_reproduce=1
--only-reproduce)
only_reproduce=1
;;
--install-android-emulator)
install_android_emulator=1
;;
esac
shift
done
Expand Down Expand Up @@ -152,6 +157,27 @@ pip install --upgrade pip
pip install --upgrade -r docker/ci/requirements.txt
pip install --upgrade -r src/local/requirements.txt

if [ $install_android_emulator ]; then
ANDROID_SDK_INSTALL_DIR=local/bin/android-sdk
ANDROID_SDK_REVISION=4333796
ANDROID_VERSION=28
ANDROID_TOOLS_BIN=$ANDROID_SDK_INSTALL_DIR/tools/bin/

# Install the Android emulator and its dependencies. Used in tests and as an
# option during Android test case reproduction.
rm -rf $ANDROID_SDK_INSTALL_DIR
mkdir $ANDROID_SDK_INSTALL_DIR
curl https://dl.google.com/android/repository/sdk-tools-linux-$ANDROID_SDK_REVISION.zip \
--output $ANDROID_SDK_INSTALL_DIR/sdk-tools-linux.zip
unzip -d $ANDROID_SDK_INSTALL_DIR $ANDROID_SDK_INSTALL_DIR/sdk-tools-linux.zip

$ANDROID_TOOLS_BIN/sdkmanager "emulator"
$ANDROID_TOOLS_BIN/sdkmanager "platform-tools" "platforms;android-$ANDROID_VERSION"
$ANDROID_TOOLS_BIN/sdkmanager "system-images;android-$ANDROID_VERSION;google_apis;x86"
$ANDROID_TOOLS_BIN/sdkmanager --licenses
$ANDROID_TOOLS_BIN/avdmanager create avd --force -n TestImage -k "system-images;android-$ANDROID_VERSION;google_apis;x86"
fi

if [ ! $only_reproduce ]; then
# Install other dependencies (e.g. bower).
nodeenv -p --prebuilt
Expand Down
20 changes: 17 additions & 3 deletions reproduce.sh
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,27 @@
CLUSTERFUZZ_CONFIG_DIR=~/.config/clusterfuzz
ROOT_DIRECTORY=$(dirname $(readlink -f "$0"))

# If we're using the emulator, make sure we install the Android SDK.
original_args="$*"
while [ "$1" != "" ]; do
case $1 in
-e)
additional_deps_args=--install-android-emulator
;;
--emulator)
additional_deps_args=--install-android-emulator
;;
esac
shift
done

mkdir -p $CLUSTERFUZZ_CONFIG_DIR
if [ ! -d $ROOT_DIRECTORY/ENV ]; then
if [ ! -d $ROOT_DIRECTORY/ENV ] || ([ $additional_deps_args ] && [ ! -d $ROOT_DIRECTORY/local/bin/android-sdk ]); then
echo "Running first time setup. This may take a while, but is only required once."
echo "You may see several password prompts to install required packages."
sleep 5
$ROOT_DIRECTORY/local/install_deps.bash --only-reproduce || { rm -rf $ROOT_DIRECTORY/ENV && exit 1; }
$ROOT_DIRECTORY/local/install_deps.bash --only-reproduce $additional_deps_args || { rm -rf $ROOT_DIRECTORY/ENV && exit 1; }
fi

source ENV/bin/activate
python $ROOT_DIRECTORY/butler.py reproduce $*
python $ROOT_DIRECTORY/butler.py reproduce $original_args
49 changes: 17 additions & 32 deletions src/local/butler/reproduce.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,11 @@
from build_management import build_manager
from datastore import data_types
from local.butler import appengine
from local.butler.reproduce_tool import android
from local.butler.reproduce_tool import config
from local.butler.reproduce_tool import errors
from local.butler.reproduce_tool import http_utils
from local.butler.reproduce_tool import prompts
from platforms.android import device
from system import archive
from system import environment
from system import new_process
Expand Down Expand Up @@ -297,35 +297,6 @@ def _get_testcase_id_from_url(testcase_url):
return testcase_id


def _prepare_environment_for_android(disable_android_setup):
"""Additional environment overrides needed to run on an Android device."""
environment.set_value('OS_OVERRIDE', 'ANDROID')

# Bail out if we don't have an Android device connected.
serial = environment.get_value('ANDROID_SERIAL')
if not serial:
# TODO(mbarbella): Handle the one-device case gracefully.
raise errors.ReproduceToolUnrecoverableError(
'This test case requires an Android device. Please set the '
'ANDROID_SERIAL environment variable and try again.')

print('Warning: this tool will make changes to settings on the connected '
'Android device with serial {serial} that could result in data '
'loss.'.format(serial=serial))
willing_to_continue = prompts.get_boolean(
'Are you sure you want to continue?')
if not willing_to_continue:
raise errors.ReproduceToolUnrecoverableError(
'Bailing out to avoid changing settings on the connected device.')

# Push the test case and build APK to the device.
apk_path = environment.get_value('APP_PATH')
device.update_build(
apk_path, should_initialize_device=not disable_android_setup)

device.push_testcases_to_device()


def _print_stacktrace(result):
"""Display the output from a test case run."""
print('#' * 80)
Expand All @@ -345,7 +316,12 @@ def _reproduce_crash(testcase_url, build_directory, iterations, disable_xvfb,

testcase = _get_testcase(testcase_id, configuration)

# Ensure that we support this test case.
# For new user uploads, we'll fail without the metadata set by analyze task.
if not testcase.platform:
raise errors.ReproduceToolUnrecoverableError(
'This test case has not yet been processed. Please try again later.')

# Ensure that we support this test case's platform.
if testcase.platform not in SUPPORTED_PLATFORMS:
raise errors.ReproduceToolUnrecoverableError(
'The reproduce tool is not yet supported on {platform}.'.format(
Expand All @@ -365,7 +341,7 @@ def _reproduce_crash(testcase_url, build_directory, iterations, disable_xvfb,
# Validate that we're running on the right platform for this test case.
platform = environment.platform().lower()
if testcase.platform == 'android' and platform == 'linux':
_prepare_environment_for_android(disable_android_setup)
android.prepare_environment(disable_android_setup)
elif testcase.platform == 'android' and platform != 'linux':
raise errors.ReproduceToolUnrecoverableError(
'The ClusterFuzz environment only supports running Android test cases '
Expand Down Expand Up @@ -423,6 +399,12 @@ def execute(args):
# Initialize fuzzing engines.
init.run()

# Prepare the emulator if needed.
emulator_process = None
if args.emulator:
print('Starting emulator...')
emulator_process = android.start_emulator()

# The current working directory may change while we're running.
absolute_build_dir = os.path.abspath(args.build_dir)
try:
Expand All @@ -432,6 +414,9 @@ def execute(args):
except errors.ReproduceToolUnrecoverableError as exception:
print(exception)
return
finally:
if emulator_process:
emulator_process.terminate()

if not result:
return
Expand Down
109 changes: 109 additions & 0 deletions src/local/butler/reproduce_tool/android.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
# Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Android emulator installation and management."""
from __future__ import print_function

import os
import time

from local.butler.reproduce_tool import errors
from local.butler.reproduce_tool import prompts
from platforms.android import adb
from platforms.android import device
from system import environment
from system import new_process

ADB_DEVICES_SEPARATOR_STRING = 'List of devices attached'
EMULATOR_RELATIVE_PATH = os.path.join('local', 'bin', 'android-sdk', 'emulator',
'emulator')


def start_emulator():
"""Return a ProcessRunner configured to start the Android emulator."""
root_dir = environment.get_value('ROOT_DIR')

runner = new_process.ProcessRunner(
os.path.join(root_dir, EMULATOR_RELATIVE_PATH),
['-avd', 'TestImage', '-writable-system', '-partition-size', '2048'])
emulator_process = runner.run()

# If we run adb commands too soon after the emulator starts, we may see
# flake or errors. Delay a short while to account for this.
# TODO(mbarbella): This is slow and flaky, but wait-for-device isn't usable if
# another device is connected (as we don't know the serial yet). Find a better
# solution.
time.sleep(30)

return emulator_process


def get_devices():
"""Get a list of all connected Android devices."""
adb_runner = new_process.ProcessRunner(adb.get_adb_path())
result = adb_runner.run_and_wait(additional_args=['devices'])

if result.return_code:
raise errors.ReproduceToolUnrecoverableError('Unable to run adb.')

# Ignore non-device lines (those before "List of devices attached").
store_devices = False
devices = []
for line in result.output.splitlines():
if line == ADB_DEVICES_SEPARATOR_STRING:
store_devices = True
continue
if not store_devices or not line:
continue

devices.append(line.split()[0])

return devices


def prepare_environment(disable_android_setup):
"""Additional environment overrides needed to run on an Android device."""
environment.set_value('OS_OVERRIDE', 'ANDROID')

# Bail out if we can't determine which Android device to use.
serial = environment.get_value('ANDROID_SERIAL')
if not serial:
devices = get_devices()
if len(devices) == 1:
serial = devices[0]
environment.set_value('ANDROID_SERIAL', serial)
elif not devices:
raise errors.ReproduceToolUnrecoverableError(
'No connected Android devices were detected. Run with the -e '
'argument to use an emulator.')
else:
raise errors.ReproduceToolUnrecoverableError(
'You have multiple Android devices or emulators connected. Please '
'set the ANDROID_SERIAL environment variable and try again.\n\n'
'Attached devices: ' + ', '.join(devices))

print('Warning: this tool will make changes to settings on the connected '
'Android device with serial {serial} that could result in data '
'loss.'.format(serial=serial))
willing_to_continue = prompts.get_boolean(
'Are you sure you want to continue?')
if not willing_to_continue:
raise errors.ReproduceToolUnrecoverableError(
'Bailing out to avoid changing settings on the connected device.')

# Push the test case and build APK to the device.
apk_path = environment.get_value('APP_PATH')
device.update_build(
apk_path, should_initialize_device=not disable_android_setup)

device.push_testcases_to_device()
2 changes: 1 addition & 1 deletion src/local/butler/reproduce_tool/http_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@

CONFIG_DIRECTORY = os.path.join(
os.path.expanduser('~'), '.config', 'clusterfuzz')
AUTHORIZATION_CACHE_FILE = os.path.join(CONFIG_DIRECTORY, 'authorization-cache')

AUTHORIZATION_CACHE_FILE = os.path.join(CONFIG_DIRECTORY, 'authorization-cache')
AUTHORIZATION_HEADER = 'x-clusterfuzz-authorization'


Expand Down

0 comments on commit fed349e

Please sign in to comment.