ci: Instrumentation tests coverage in ci.

2025-03-16 15:55:10 +01:00
parent 02d503e03a
commit f5d1a995a3
13 changed files with 436 additions and 151 deletions
--- a/androidApp/build.gradle.kts
+++ b/androidApp/build.gradle.kts
@ -95,7 +95,7 @@ android {

        // tests
        testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
-        testInstrumentationRunnerArguments["clearPackageData"] = "true"
+        testInstrumentationRunnerArguments["useTestStorageService"] = "true"
    }
    packaging {
        resources {
@ -109,6 +109,11 @@ android {
            proguardFiles(getDefaultProguardFile("proguard-android.txt"), "proguard-rules.pro")
        }
        getByName("debug") {
+            isTestCoverageEnabled = true
+            enableAndroidTestCoverage = true
+            installation {
+                installOptions("-g", "-r")
+            }
        }
    }
    flavorDimensions.add("build")
@ -121,7 +126,6 @@ android {
    namespace = "bou.amine.apps.readerforselfossv2.android"
    testOptions {
        animationsDisabled = true
-        execution = "ANDROIDX_TEST_ORCHESTRATOR"
        unitTests {
            isIncludeAndroidResources = true
        }
@ -154,8 +158,8 @@ dependencies {
    implementation("androidx.multidex:multidex:2.0.1")

    // About
-    implementation("com.mikepenz:aboutlibraries-core:10.5.1")
-    implementation("com.mikepenz:aboutlibraries:10.5.1")
+    implementation("com.mikepenz:aboutlibraries-core:11.6.3")
+    implementation("com.mikepenz:aboutlibraries:11.6.3")

    // Material-ish things
    implementation("com.ashokvarma.android:bottom-navigation-bar:2.2.0")
@ -197,14 +201,15 @@ dependencies {
    testImplementation("io.mockk:mockk:1.13.14")
    testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.10.1")
    implementation("org.jetbrains.kotlinx:kotlinx-datetime:0.6.1")
-    androidTestImplementation("androidx.test:runner:1.6.2")
-    androidTestImplementation("androidx.test:rules:1.6.1")
+    androidTestImplementation("androidx.test:runner:1.7.0-alpha01")
+    androidTestImplementation("androidx.test:rules:1.7.0-alpha01")
    androidTestImplementation("androidx.test.espresso:espresso-core:3.6.1")
    implementation("androidx.test.espresso:espresso-idling-resource:3.6.1")
    androidTestImplementation("androidx.test.ext:junit-ktx:1.2.1")
-    androidTestUtil("androidx.test:orchestrator:1.5.1")
+    androidTestUtil("androidx.test.services:test-services:1.6.0-alpha02")
    testImplementation("org.robolectric:robolectric:4.14.1")
-    testImplementation("androidx.test:core-ktx:1.6.1")
+    testImplementation("androidx.test:core-ktx:1.7.0-alpha01")
+    androidTestImplementation("androidx.test.uiautomator:uiautomator:2.3.0")

    implementation("ch.acra:acra-http:$acraVersion")
    implementation("ch.acra:acra-toast:$acraVersion")
@ -227,6 +232,7 @@ tasks.withType<Test> {
 }

 aboutLibraries {
+    excludeFields = arrayOf("generated")
    offlineMode = true
    fetchRemoteLicense = false
    fetchRemoteFunding = false
@ -235,3 +241,33 @@ aboutLibraries {
    duplicationMode = com.mikepenz.aboutlibraries.plugin.DuplicateMode.MERGE
    duplicationRule = com.mikepenz.aboutlibraries.plugin.DuplicateRule.GROUP
 }
+
+// Screenshot failure handling
+val reportsDirectory = file("$buildDir/reports/androidTests/connected")
+
+val clearScreenshotsTask =
+    tasks.register<Exec>("clearScreenshots") {
+        println("AMINE : clear")
+        commandLine = listOf("adb", "shell", "rm", "-r", "/storage/emulated/0//Pictures/selfoss_tests")
+    }
+
+val createScreenshotDirectoryTask =
+    tasks.register<Exec>("createScreenshotDirectory") {
+        println("AMINE : create directory")
+        group = "reporting"
+        commandLine = listOf("adb", "shell", "mkdir", "-p", "/storage/emulated/0//Pictures/selfoss_tests")
+    }
+
+tasks.register<Exec>("fetchScreenshots") {
+    println("AMINE : fetch")
+    group = "reporting"
+    executable(android.adbExecutable.toString())
+    commandLine = listOf("adb", "pull", "/storage/emulated/0//Pictures/selfoss_tests/.", reportsDirectory.toString())
+
+    finalizedBy(clearScreenshotsTask)
+    dependsOn(createScreenshotDirectoryTask)
+
+    doFirst {
+        reportsDirectory.mkdirs()
+    }
+}
--- a/androidApp/src/androidTest/kotlin/bou/amine/apps/readerforselfossv2/android/1-LoginActivityTest.kt
+++ b/androidApp/src/androidTest/kotlin/bou/amine/apps/readerforselfossv2/android/1-LoginActivityTest.kt
@ -15,13 +15,17 @@ import androidx.test.filters.LargeTest
 import bou.amine.apps.readerforselfossv2.android.testing.CountingIdlingResourceSingleton
 import org.junit.After
 import org.junit.Before
+import org.junit.FixMethodOrder
 import org.junit.Rule
 import org.junit.Test
 import org.junit.runner.RunWith
+import org.junit.runners.MethodSorters

+@Suppress("ktlint:standard:class-naming", "detekt:ClassNaming")
@RunWith(AndroidJUnit4::class)
+@FixMethodOrder(MethodSorters.NAME_ASCENDING)
@LargeTest
-class LoginActivityTest {
+class `1-LoginActivityTest` : WithANRException() {
    @get:Rule
    val activityRule = ActivityScenarioRule(LoginActivity::class.java)

@ -40,7 +44,7 @@ class LoginActivityTest {
    }

    @Test
-    fun viewIsInitialized() {
+    fun `1-viewIsInitialized`() {
        onView(withId(R.id.urlView)).check(matches(isDisplayed()))
        onView(withId(R.id.selfSigned))
            .check(matches(isDisplayed()))
@ -57,28 +61,28 @@ class LoginActivityTest {
    }

    @Test
-    fun urlError() {
+    fun `2-urlError`() {
        performLogin("10.0.2.2:8888")
        onView(withId(R.id.urlView)).perform(click())
        onView(withId(R.id.urlView)).check(matches(withError(R.string.login_url_problem)))
    }

    @Test
-    fun connectError() {
-        performLogin("http://10.0.2.2:8889")
-        onView(withId(R.id.urlView)).perform(click())
-        onView(withId(R.id.urlView)).check(matches(withError(R.string.wrong_infos)))
-    }
-
-    @Test
-    fun urlSlashError() {
+    fun `3-urlSlashError`() {
        performLogin("https://google.fr/toto")
        onView(withId(R.id.urlView)).perform(click())
        onView(withId(R.id.urlView)).check(matches(withError(R.string.login_url_problem)))
    }

    @Test
-    fun multiError() {
+    fun `4-connectError`() {
+        performLogin("http://10.0.2.2:8889")
+        onView(withId(R.id.urlView)).perform(click())
+        onView(withId(R.id.urlView)).check(matches(withError(R.string.wrong_infos)))
+    }
+
+    @Test
+    fun `5-multiError`() {
        onView(withId(R.id.signInButton)).perform(click())
        onView(withId(R.id.signInButton)).perform(click())
        onView(withId(R.id.signInButton)).perform(click())
@ -86,8 +90,9 @@ class LoginActivityTest {
    }

    @Test
-    fun connect() {
+    fun `6-connect`() {
        performLogin()
        onView(withText(R.string.gdpr_dialog_title)).check(matches(isDisplayed()))
+        onView(withText("OK")).perform(click())
    }
 }
--- a/androidApp/src/androidTest/kotlin/bou/amine/apps/readerforselfossv2/android/2-HomeActivityTest.kt
+++ b/androidApp/src/androidTest/kotlin/bou/amine/apps/readerforselfossv2/android/2-HomeActivityTest.kt
@ -15,21 +15,19 @@ import androidx.test.ext.junit.rules.ActivityScenarioRule
 import androidx.test.ext.junit.runners.AndroidJUnit4
 import androidx.test.filters.LargeTest
 import org.hamcrest.CoreMatchers.not
-import org.junit.Before
+import org.junit.FixMethodOrder
 import org.junit.Rule
 import org.junit.Test
 import org.junit.runner.RunWith
+import org.junit.runners.MethodSorters

+@Suppress("ktlint:standard:class-naming", "detekt:ClassNaming")
@RunWith(AndroidJUnit4::class)
+@FixMethodOrder(MethodSorters.NAME_ASCENDING)
@LargeTest
-class HomeActivityTest {
+class `2-HomeActivityTest` : WithANRException() {
    @get:Rule
-    val activityRule = ActivityScenarioRule(LoginActivity::class.java)
-
-    @Before
-    fun init() {
-        loginAndInitHome()
-    }
+    val activityRule = ActivityScenarioRule(HomeActivity::class.java)

    @Test
    fun testMenu() {
--- a/androidApp/src/androidTest/kotlin/bou/amine/apps/readerforselfossv2/android/3-SettingsActivityTest.kt
+++ b/androidApp/src/androidTest/kotlin/bou/amine/apps/readerforselfossv2/android/3-SettingsActivityTest.kt
@ -19,9 +19,11 @@ import org.junit.runner.RunWith

@RunWith(AndroidJUnit4::class)
@LargeTest
-class SettingsActivityTest {
+@Suppress("ktlint:standard:class-naming", "detekt:ClassNaming")
+class `3-SettingsActivityTest` : WithANRException() {
    @get:Rule
-    val activityRule = ActivityScenarioRule(LoginActivity::class.java)
+    val activityRule = ActivityScenarioRule(HomeActivity::class.java)
+
    lateinit var context: Context

    @Before
@ -29,7 +31,6 @@ class SettingsActivityTest {
        activityRule.scenario.onActivity { activity ->
            context = activity.window.context
        }
-        loginAndInitHome()
        openMenu()
        onView(withText(R.string.title_activity_settings)).perform(click())
    }
@ -68,6 +69,9 @@ class SettingsActivityTest {
        changeAndSaveSetting("", "10") {
            onView(withText(R.string.pref_api_timeout)).perform(click())
        }
+        changeAndSaveSetting("", "60") {
+            onView(withText(R.string.pref_api_timeout)).perform(click())
+        }
    }

    @Test
--- a/androidApp/src/androidTest/kotlin/bou/amine/apps/readerforselfossv2/android/4-SettingsActivityGeneralTest.kt
+++ b/androidApp/src/androidTest/kotlin/bou/amine/apps/readerforselfossv2/android/4-SettingsActivityGeneralTest.kt
@ -22,19 +22,22 @@ import androidx.test.filters.LargeTest
 import org.hamcrest.CoreMatchers.allOf
 import org.hamcrest.CoreMatchers.not
 import org.junit.Before
+import org.junit.FixMethodOrder
 import org.junit.Rule
 import org.junit.Test
 import org.junit.runner.RunWith
+import org.junit.runners.MethodSorters

+@Suppress("ktlint:standard:class-naming", "detekt:ClassNaming")
@RunWith(AndroidJUnit4::class)
+@FixMethodOrder(MethodSorters.NAME_ASCENDING)
@LargeTest
-class SettingsActivityGeneralTest {
+class `4-SettingsActivityGeneralTest` : WithANRException() {
    @get:Rule
-    val activityRule = ActivityScenarioRule(LoginActivity::class.java)
+    val activityRule = ActivityScenarioRule(HomeActivity::class.java)

    @Before
    fun init() {
-        loginAndInitHome()
        openActionBarOverflowOrOptionsMenu(
            ApplicationProvider.getApplicationContext(),
        )
--- a/androidApp/src/androidTest/kotlin/bou/amine/apps/readerforselfossv2/android/5-SettingsActivityReaderTest.kt
+++ b/androidApp/src/androidTest/kotlin/bou/amine/apps/readerforselfossv2/android/5-SettingsActivityReaderTest.kt
@ -1,29 +1,32 @@
 package bou.amine.apps.readerforselfossv2.android

 import android.content.Context
-import androidx.test.core.app.ApplicationProvider
 import androidx.test.espresso.Espresso.onView
-import androidx.test.espresso.Espresso.openActionBarOverflowOrOptionsMenu
+import androidx.test.espresso.action.ViewActions
 import androidx.test.espresso.action.ViewActions.click
 import androidx.test.espresso.assertion.ViewAssertions.matches
 import androidx.test.espresso.matcher.ViewMatchers.isChecked
 import androidx.test.espresso.matcher.ViewMatchers.isDisplayed
+import androidx.test.espresso.matcher.ViewMatchers.isRoot
 import androidx.test.espresso.matcher.ViewMatchers.withText
 import androidx.test.ext.junit.rules.ActivityScenarioRule
 import androidx.test.ext.junit.runners.AndroidJUnit4
 import androidx.test.filters.LargeTest
+import bou.amine.apps.readerforselfossv2.android.settings.SettingsActivity
 import org.hamcrest.CoreMatchers.allOf
 import org.hamcrest.CoreMatchers.not
+import org.junit.After
 import org.junit.Before
 import org.junit.Rule
 import org.junit.Test
 import org.junit.runner.RunWith

+@Suppress("ktlint:standard:class-naming", "detekt:ClassNaming")
@RunWith(AndroidJUnit4::class)
@LargeTest
-class SettingsActivityReaderTest {
+class `5-SettingsActivityReaderTest` : WithANRException() {
    @get:Rule
-    val activityRule = ActivityScenarioRule(LoginActivity::class.java)
+    val activityRule = ActivityScenarioRule(SettingsActivity::class.java)

    lateinit var context: Context

@ -32,14 +35,14 @@ class SettingsActivityReaderTest {
        activityRule.scenario.onActivity { activity ->
            context = activity.window.context
        }
-        loginAndInitHome()
-        openActionBarOverflowOrOptionsMenu(
-            ApplicationProvider.getApplicationContext(),
-        )
-        onView(withText(R.string.title_activity_settings)).perform(click())
        onView(withText(R.string.pref_header_viewer)).perform(click())
    }

+    @After
+    fun back() {
+        onView(isRoot()).perform(ViewActions.pressBack())
+    }
+
    @Test
    fun testReader() {
        onView(withSettingsCheckboxFrame(R.string.pref_switch_actions_pager_scroll)).check(
--- a/androidApp/src/androidTest/kotlin/bou/amine/apps/readerforselfossv2/android/6-SettingsActivityOfflineTest.kt
+++ b/androidApp/src/androidTest/kotlin/bou/amine/apps/readerforselfossv2/android/6-SettingsActivityOfflineTest.kt
@ -1,31 +1,34 @@
 package bou.amine.apps.readerforselfossv2.android

 import android.content.Context
-import androidx.test.core.app.ApplicationProvider
 import androidx.test.espresso.Espresso.onView
-import androidx.test.espresso.Espresso.openActionBarOverflowOrOptionsMenu
+import androidx.test.espresso.action.ViewActions
 import androidx.test.espresso.action.ViewActions.click
 import androidx.test.espresso.assertion.ViewAssertions.matches
 import androidx.test.espresso.matcher.ViewMatchers.isChecked
 import androidx.test.espresso.matcher.ViewMatchers.isDisplayed
 import androidx.test.espresso.matcher.ViewMatchers.isEnabled
 import androidx.test.espresso.matcher.ViewMatchers.isNotEnabled
+import androidx.test.espresso.matcher.ViewMatchers.isRoot
 import androidx.test.espresso.matcher.ViewMatchers.withText
 import androidx.test.ext.junit.rules.ActivityScenarioRule
 import androidx.test.ext.junit.runners.AndroidJUnit4
 import androidx.test.filters.LargeTest
+import bou.amine.apps.readerforselfossv2.android.settings.SettingsActivity
 import org.hamcrest.CoreMatchers.allOf
 import org.hamcrest.CoreMatchers.not
+import org.junit.After
 import org.junit.Before
 import org.junit.Rule
 import org.junit.Test
 import org.junit.runner.RunWith

+@Suppress("ktlint:standard:class-naming", "detekt:ClassNaming")
@RunWith(AndroidJUnit4::class)
@LargeTest
-class SettingsActivityOfflineTest {
+class `6-SettingsActivityOfflineTest` : WithANRException() {
    @get:Rule
-    val activityRule = ActivityScenarioRule(LoginActivity::class.java)
+    val activityRule = ActivityScenarioRule(SettingsActivity::class.java)

    lateinit var context: Context

@ -34,14 +37,14 @@ class SettingsActivityOfflineTest {
        activityRule.scenario.onActivity { activity ->
            context = activity.window.context
        }
-        loginAndInitHome()
-        openActionBarOverflowOrOptionsMenu(
-            ApplicationProvider.getApplicationContext(),
-        )
-        onView(withText(R.string.title_activity_settings)).perform(click())
        onView(withText(R.string.pref_header_offline)).perform(click())
    }

+    @After
+    fun back() {
+        onView(isRoot()).perform(ViewActions.pressBack())
+    }
+
    @Suppress("detekt:LongMethod")
    @Test
    fun testOffline() {
--- a/androidApp/src/androidTest/kotlin/bou/amine/apps/readerforselfossv2/android/7-SourcesActivityTest.kt
+++ b/androidApp/src/androidTest/kotlin/bou/amine/apps/readerforselfossv2/android/7-SourcesActivityTest.kt
@ -21,11 +21,12 @@ import org.junit.Test
 import org.junit.runner.RunWith
 import java.util.UUID

+@Suppress("ktlint:standard:class-naming", "detekt:ClassNaming")
@RunWith(AndroidJUnit4::class)
@LargeTest
-class SourcesActivityTest {
+class `7-SourcesActivityTest` : WithANRException() {
    @get:Rule
-    val activityRule = ActivityScenarioRule(LoginActivity::class.java)
+    val activityRule = ActivityScenarioRule(HomeActivity::class.java)

    lateinit var sourceName: String

@ -33,7 +34,6 @@ class SourcesActivityTest {
    fun init() {
        sourceName = UUID.randomUUID().toString().substring(0, 15)

-        loginAndInitHome()
        goToSources()
    }

@ -75,10 +75,4 @@ class SourcesActivityTest {
        onView(withId(android.R.id.button1)).perform(click())
        onView(withText(sourceName)).check(doesNotExist())
    }
-
-    private fun goToSources() {
-        openMenu()
-        onView(withText(R.string.menu_home_sources))
-            .perform(click())
-    }
 }
--- a/androidApp/src/androidTest/kotlin/bou/amine/apps/readerforselfossv2/android/CommonTests.kt
+++ b/androidApp/src/androidTest/kotlin/bou/amine/apps/readerforselfossv2/android/CommonTests.kt
@ -1,7 +1,12 @@
 package bou.amine.apps.readerforselfossv2.android

 import android.content.Context
+import android.graphics.Bitmap
+import android.os.Environment.DIRECTORY_PICTURES
+import android.os.Environment.getExternalStoragePublicDirectory
+import android.util.Log
 import androidx.annotation.ArrayRes
+import androidx.test.espresso.Espresso
 import androidx.test.espresso.Espresso.onData
 import androidx.test.espresso.Espresso.onView
 import androidx.test.espresso.action.ViewActions.click
@ -9,29 +14,36 @@ import androidx.test.espresso.action.ViewActions.replaceText
 import androidx.test.espresso.action.ViewActions.typeTextIntoFocusedView
 import androidx.test.espresso.assertion.ViewAssertions.doesNotExist
 import androidx.test.espresso.assertion.ViewAssertions.matches
+import androidx.test.espresso.base.DefaultFailureHandler
 import androidx.test.espresso.matcher.ViewMatchers.isChecked
 import androidx.test.espresso.matcher.ViewMatchers.isDisplayed
 import androidx.test.espresso.matcher.ViewMatchers.isNotChecked
 import androidx.test.espresso.matcher.ViewMatchers.withId
 import androidx.test.espresso.matcher.ViewMatchers.withText
+import androidx.test.platform.app.InstrumentationRegistry.getInstrumentation
+import androidx.test.uiautomator.UiDevice
+import androidx.test.uiautomator.UiSelector
 import org.hamcrest.CoreMatchers.allOf
 import org.hamcrest.Matchers.hasToString
+import org.junit.BeforeClass
+import java.io.BufferedOutputStream
+import java.io.File
+import java.io.FileOutputStream
+import java.io.IOException
+import java.util.Locale
+
+val defaultUrl = (System.getenv("SELFOSS_URL") ?: "").ifEmpty { "http://10.0.2.2:8888" }

 fun performLogin(someUrl: String? = null) {
+    Log.i("AUTOMATION", "The url used will be ${if (!someUrl.isNullOrEmpty()) someUrl else defaultUrl}")
    onView(withId(R.id.urlView)).perform(click()).perform(
        typeTextIntoFocusedView(
-            if (!someUrl.isNullOrEmpty()) someUrl else "http://10.0.2.2:8888",
+            if (!someUrl.isNullOrEmpty()) someUrl else defaultUrl,
        ),
    )
    onView(withId(R.id.signInButton)).perform(click())
 }

-fun loginAndInitHome() {
-    performLogin()
-    onView(withText(R.string.gdpr_dialog_title)).check(matches(isDisplayed()))
-    onView(withText("OK")).perform(click())
-}
-
 fun changeAndCancelSetting(
    oldValue: String,
    newValue: String,
@ -97,6 +109,12 @@ fun testPreferencesFromArray(
    }
 }

+fun goToSources() {
+    openMenu()
+    onView(withText(R.string.menu_home_sources))
+        .perform(click())
+}
+
 fun testAddSourceWithUrl(
    url: String,
    sourceName: String,
@ -119,3 +137,87 @@ fun testAddSourceWithUrl(
        .perform(click())
    onView(withText(sourceName)).check(matches(isDisplayed()))
 }
+
+@Suppress("detekt:UtilityClassWithPublicConstructor")
+open class WithANRException {
+    companion object {
+        private var testNumber = 0
+
+        // Running count of the number of Android Not Responding dialogues to prevent endless dismissal.
+        private var anrCount = 0
+
+        // `RootViewWithoutFocusException` class is private, need to match the message (instead of using type matching).
+        private val rootViewWithoutFocusExceptionMsg =
+            java.lang.String.format(
+                Locale.ROOT,
+                "Waited for the root of the view hierarchy to have " +
+                    "window focus and not request layout for 10 seconds. If you specified a non " +
+                    "default root matcher, it may be picking a root that never takes focus. " +
+                    "Root:",
+            )
+
+        private fun handleAnrDialogue() {
+            val device = UiDevice.getInstance(getInstrumentation())
+            // If running the device in English Locale
+            val waitButton = device.findObject(UiSelector().textContains("wait"))
+            if (waitButton.exists()) waitButton.click()
+        }
+
+        @JvmStatic
+        @BeforeClass
+        fun setUpHandler() {
+            Espresso.setFailureHandler { error, viewMatcher ->
+
+                takeScreenshot("test-failures/", testNumber.toString())
+                if (error.message!!.contains(rootViewWithoutFocusExceptionMsg) && anrCount < 3) {
+                    anrCount++
+                    handleAnrDialogue()
+                } else { // chain all failures down to the default espresso handler
+                    DefaultFailureHandler(getInstrumentation().targetContext).handle(error, viewMatcher)
+                }
+                testNumber++
+            }
+        }
+    }
+}
+
+fun takeScreenshot(
+    parentFolderPath: String = "",
+    screenShotName: String,
+) {
+    Log.d("Screenshots", "Taking screenshot of '$screenShotName'")
+    try {
+        val bitmap = getInstrumentation().uiAutomation.takeScreenshot()
+
+        val folder =
+            File(
+                File(
+                    getExternalStoragePublicDirectory(DIRECTORY_PICTURES),
+                    "selfoss_tests",
+                ).absolutePath,
+                "screenshots/$parentFolderPath",
+            )
+        if (!folder.exists()) {
+            folder.mkdirs()
+        }
+
+        var out: BufferedOutputStream? = null
+        try {
+            out = BufferedOutputStream(FileOutputStream(folder.path + "/" + screenShotName + ".png"))
+            bitmap.compress(Bitmap.CompressFormat.PNG, 100, out)
+            Log.d("Screenshots", "Screenshot taken")
+        } catch (e: IOException) {
+            Log.e("Screenshots", "Could not save the screenshot", e)
+        } finally {
+            if (out != null) {
+                try {
+                    out.close()
+                } catch (e: IOException) {
+                    Log.e("Screenshots", "Could not save the screenshot", e)
+                }
+            }
+        }
+    } catch (ex: IOException) {
+        Log.e("Screenshots", "Could not take the screenshot", ex)
+    }
+}