devops: add bot to test Android (#4693)

Author: mxschmitt

.github/workflows/tests.yml

@@ -231,3 +231,29 @@ jobs:
       with:
         name: video-${{ matrix.browser }}-linux-test-results
         path: test-results
+  test_android:
+    name: Android on macOS
+    runs-on: macos-10.15
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-node@v1
+      with:
+        node-version: 14
+    - uses: microsoft/playwright-github-action@v1
+    - run: npm ci
+    - run: npm run build
+    - name: Create Android Emulator
+      run: utils/avd_recreate.sh
+    - name: Start Android Emulator
+      run: utils/avd_start.sh
+    - run: npx folio test/android -p browserName=chromium --workers=1 --forbid-only --global-timeout=5400000 --retries=3 --reporter=dot,json
+      env:
+        FOLIO_JSON_OUTPUT_NAME: "test-results/report.json"
+        PW_ANDROID_TESTS: 1
+    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
+      if: always() && github.ref == 'refs/heads/master'
+    - uses: actions/upload-artifact@v1
+      if: ${{ always() }}
+      with:
+        name: android-test-results
+        path: test-results

test/android/device.spec.ts

@@ -20,7 +20,7 @@ const { it, expect } = folio;
 if (process.env.PW_ANDROID_TESTS) {
   it('should run ADB shell commands', async function({ device }) {
     const output = await device.shell('echo 123');
-    expect(output).toBe('123\n');
+    expect(output.toString()).toBe('123\n');
   });
 
   it('should open a ADB socket', async function({ device }) {

utils/avd_install.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -e
+
 SDKDIR=$PWD/.android-sdk
 export ANDROID_SDK_ROOT=${SDKDIR}
 export ANDROID_HOME=${SDKDIR}
@@ -10,18 +12,18 @@ mkdir ${SDKDIR}/cmdline-tools
 
 echo Downloading Android SDK...
 cd ${SDKDIR}/cmdline-tools
-curl https://dl.google.com/android/repository/commandlinetools-mac-6858069_latest.zip -o commandlinetools-mac-6858069_latest.zip
-unzip commandlinetools-mac-6858069_latest.zip
+COMMAND_LINE_TOOLS_ZIP=${SDKDIR}/commandlinetools.zip
+curl https://dl.google.com/android/repository/commandlinetools-mac-6858069_latest.zip -o ${COMMAND_LINE_TOOLS_ZIP}
+unzip ${COMMAND_LINE_TOOLS_ZIP}
+rm ${COMMAND_LINE_TOOLS_ZIP}
 mv cmdline-tools latest
+ln -s ${SDKDIR}/cmdline-tools/latest ${SDKDIR}/tools
 
 echo Installing emulator...
-yes | ${SDKDIR}/cmdline-tools/latest/bin/sdkmanager platform-tools emulator
-
-echo Installing system image...
-${SDKDIR}/cmdline-tools/latest/bin/sdkmanager "system-images;android-30;google_apis;x86"
+yes | ${ANDROID_HOME}/tools/bin/sdkmanager --install platform-tools emulator
 
 echo Installing platform SDK...
-${SDKDIR}/cmdline-tools/latest/bin/sdkmanager "platforms;android-30"
+yes | ${ANDROID_HOME}/tools/bin/sdkmanager --install "platforms;android-30"
 
 echo Starting ADB...
-${SDKDIR}/platform-tools/adb devices
+${ANDROID_HOME}/platform-tools/adb devices

utils/avd_recreate.sh

@@ -1,9 +1,14 @@
 #!/bin/bash
 
-SDKDIR=$PWD/.android-sdk
-export ANDROID_SDK_ROOT=${SDKDIR}
-export ANDROID_HOME=${SDKDIR}
-export ANDROID_AVD_HOME=${SDKDIR}/avd
+set -e
 
-${SDKDIR}/cmdline-tools/latest/bin/avdmanager delete avd --name android30
-echo -ne '\n' | ${SDKDIR}/cmdline-tools/latest/bin/avdmanager create avd --name android30 --device pixel_4_xl --package "system-images;android-30;google_apis;x86"
+if [[ -z "${ANDROID_HOME}" ]]; then
+    export SDKDIR=$PWD/.android-sdk
+    export ANDROID_HOME=${SDKDIR}
+    export ANDROID_SDK_ROOT=${SDKDIR}
+fi
+
+${ANDROID_HOME}/tools/bin/avdmanager delete avd --name android30
+echo "y" | ${ANDROID_HOME}/tools/bin/sdkmanager --install "system-images;android-30;google_apis;x86"
+echo "no" | ${ANDROID_HOME}/tools/bin/avdmanager create avd --force --name android30 --device "Nexus 5X" -k 'system-images;android-30;google_apis;x86'
+${ANDROID_HOME}/emulator/emulator -list-avds

utils/avd_start.sh

@@ -1,8 +1,18 @@
 #!/bin/bash
 
-SDKDIR=$PWD/.android-sdk
-export ANDROID_SDK_ROOT=${SDKDIR}
-export ANDROID_HOME=${SDKDIR}
-export ANDROID_AVD_HOME=${SDKDIR}/avd
+set -e
 
-${SDKDIR}/emulator/emulator -avd android30 -gpu swiftshader_indirect
+if [[ -z "${ANDROID_HOME}" ]]; then
+    SDKDIR=$PWD/.android-sdk
+    export ANDROID_HOME=${SDKDIR}
+    export ANDROID_SDK_ROOT=${SDKDIR}
+fi
+
+echo "Killing previous emulators"
+adb devices | grep emulator | cut -f1 | while read line; do adb -s $line emu kill; done
+
+echo "Starting emulator"
+nohup ${ANDROID_HOME}/emulator/emulator -avd android30 -no-audio -no-snapshot -no-window -gpu swiftshader_indirect &
+${ANDROID_HOME}/platform-tools/adb wait-for-device shell 'while [[ -z $(getprop sys.boot_completed | tr -d '\r') ]]; do sleep 1; done; input keyevent 82'
+${ANDROID_HOME}/platform-tools/adb devices
+echo "Emulator started"