Analyzer: Speed up scans further

Introduce new gateway method to just look op sizes like `du`. More efficient than doing complete lookups that are then summed.
GHT66 · Jun 7, 2024 · 948b8da · 948b8da
1 parent 022338a
commit 948b8da
Show file tree

Hide file tree

Showing 13 changed files with 208 additions and 22 deletions.
diff --git a/app-common-io/src/main/aidl/eu/darken/sdmse/common/files/local/ipc/FileOpsConnection.aidl b/app-common-io/src/main/aidl/eu/darken/sdmse/common/files/local/ipc/FileOpsConnection.aidl
@@ -34,6 +34,8 @@ interface FileOpsConnection {
 
     RemoteInputStream walkStream(in LocalPath path, in List<String> pathDoesNotContain);
 
+    long du(in LocalPath path);
+
     boolean createSymlink(in LocalPath linkPath, in LocalPath targetPath);
 
     boolean setModifiedAt(in LocalPath path, in long modifiedAt);

diff --git a/app-common-io/src/main/java/eu/darken/sdmse/common/files/APathExtensions.kt b/app-common-io/src/main/java/eu/darken/sdmse/common/files/APathExtensions.kt
@@ -2,12 +2,12 @@ package eu.darken.sdmse.common.files
 
 import eu.darken.sdmse.common.debug.logging.Logging.Priority.VERBOSE
 import eu.darken.sdmse.common.debug.logging.log
-import eu.darken.sdmse.common.files.local.*
+import eu.darken.sdmse.common.files.local.LocalPath
 import eu.darken.sdmse.common.files.local.crumbsTo
 import eu.darken.sdmse.common.files.local.isAncestorOf
 import eu.darken.sdmse.common.files.local.isParentOf
 import eu.darken.sdmse.common.files.local.startsWith
-import eu.darken.sdmse.common.files.saf.*
+import eu.darken.sdmse.common.files.saf.SAFPath
 import eu.darken.sdmse.common.files.saf.crumbsTo
 import eu.darken.sdmse.common.files.saf.isAncestorOf
 import eu.darken.sdmse.common.files.saf.isParentOf
@@ -18,7 +18,7 @@ import okio.Source
 import java.io.File
 import java.io.IOException
 import java.time.Instant
-import java.util.*
+import java.util.Collections
 import eu.darken.sdmse.common.files.local.removePrefix as removePrefixLocalPath
 import eu.darken.sdmse.common.files.saf.removePrefix as removePrefixSafPath
 
@@ -47,6 +47,14 @@ suspend fun <P : APath, PL : APathLookup<P>, PLE : APathLookupExtended<P>, GT :
     return gateway.walk(this, options)
 }
 
+
+suspend fun <P : APath, PL : APathLookup<P>, PLE : APathLookupExtended<P>, GT : APathGateway<P, PL, PLE>> P.du(
+    gateway: GT,
+    options: APathGateway.DuOptions<P, PL> = APathGateway.DuOptions()
+): Long {
+    return gateway.du(this, options)
+}
+
 suspend fun <T : APath> T.exists(gateway: APathGateway<T, out APathLookup<T>, out APathLookupExtended<T>>): Boolean {
     return gateway.exists(this)
 }

diff --git a/app-common-io/src/main/java/eu/darken/sdmse/common/files/APathGateway.kt b/app-common-io/src/main/java/eu/darken/sdmse/common/files/APathGateway.kt
@@ -38,6 +38,15 @@ interface APathGateway<
             get() = onFilter == null && onError == null
     }
 
+    suspend fun du(
+        path: P,
+        options: DuOptions<P, PLU> = DuOptions()
+    ): Long
+
+    data class DuOptions<P : APath, PLU : APathLookup<P>>(
+        val abortOnError: Boolean = false,
+    )
+
     suspend fun exists(path: P): Boolean
 
     suspend fun canWrite(path: P): Boolean

diff --git a/app-common-io/src/main/java/eu/darken/sdmse/common/files/APathLookupExtensions.kt b/app-common-io/src/main/java/eu/darken/sdmse/common/files/APathLookupExtensions.kt
@@ -21,6 +21,11 @@ suspend fun <P : APath, PL : APathLookup<P>, PLE : APathLookupExtended<P>, GT :
     options: APathGateway.WalkOptions<P, PL> = APathGateway.WalkOptions()
 ): Flow<PL> = lookedUp.walk(gateway, options)
 
+suspend fun <P : APath, PL : APathLookup<P>, PLE : APathLookupExtended<P>, GT : APathGateway<P, PL, PLE>> PL.du(
+    gateway: GT,
+    options: APathGateway.DuOptions<P, PL> = APathGateway.DuOptions()
+): Long = lookedUp.du(gateway, options)
+
 suspend fun <P : APath, PL : APathLookup<P>> PL.exists(
     gateway: APathGateway<P, out APathLookup<P>, out APathLookupExtended<P>>
 ): Boolean = lookedUp.exists(gateway)

diff --git a/app-common-io/src/main/java/eu/darken/sdmse/common/files/GatewaySwitch.kt b/app-common-io/src/main/java/eu/darken/sdmse/common/files/GatewaySwitch.kt
@@ -143,6 +143,14 @@ class GatewaySwitch @Inject constructor(
         return useGateway(path) { walk(path, options) }
     }
 
+
+    override suspend fun du(
+        path: APath,
+        options: APathGateway.DuOptions<APath, APathLookup<APath>>
+    ): Long {
+        return useGateway(path) { du(path, options) }
+    }
+
     override suspend fun listFiles(path: APath): Collection<APath> {
         return useGateway(path) { listFiles(path) }
     }

diff --git a/app-common-io/src/main/java/eu/darken/sdmse/common/files/local/EscalatingWalker.kt b/app-common-io/src/main/java/eu/darken/sdmse/common/files/local/EscalatingWalker.kt
@@ -21,10 +21,10 @@ import java.util.LinkedList
 /**
  * Prevents unnecessary lookups in Mode.NORMAL for nested directories
  */
-class EscalatingWalker constructor(
+class EscalatingWalker(
     private val gateway: LocalGateway,
     private val start: LocalPath,
-    private val options: APathGateway.WalkOptions<LocalPath, LocalPathLookup>,
+    private val options: APathGateway.WalkOptions<LocalPath, LocalPathLookup> = APathGateway.WalkOptions()
 ) : AbstractFlow<LocalPathLookup>() {
     private val tag = "$TAG#${hashCode()}"
 

diff --git a/app-common-io/src/main/java/eu/darken/sdmse/common/files/local/LocalGateway.kt b/app-common-io/src/main/java/eu/darken/sdmse/common/files/local/LocalGateway.kt
@@ -3,11 +3,19 @@ package eu.darken.sdmse.common.files.local
 import eu.darken.sdmse.common.coroutine.AppScope
 import eu.darken.sdmse.common.coroutine.DispatcherProvider
 import eu.darken.sdmse.common.debug.Bugs
-import eu.darken.sdmse.common.debug.logging.Logging.Priority.*
+import eu.darken.sdmse.common.debug.logging.Logging.Priority.INFO
+import eu.darken.sdmse.common.debug.logging.Logging.Priority.VERBOSE
+import eu.darken.sdmse.common.debug.logging.Logging.Priority.WARN
 import eu.darken.sdmse.common.debug.logging.asLog
 import eu.darken.sdmse.common.debug.logging.log
 import eu.darken.sdmse.common.debug.logging.logTag
-import eu.darken.sdmse.common.files.*
+import eu.darken.sdmse.common.files.APathGateway
+import eu.darken.sdmse.common.files.Ownership
+import eu.darken.sdmse.common.files.Permissions
+import eu.darken.sdmse.common.files.ReadException
+import eu.darken.sdmse.common.files.WriteException
+import eu.darken.sdmse.common.files.asFile
+import eu.darken.sdmse.common.files.callbacks
 import eu.darken.sdmse.common.files.core.local.createSymlink
 import eu.darken.sdmse.common.files.core.local.isReadable
 import eu.darken.sdmse.common.files.core.local.listFiles2
@@ -26,14 +34,15 @@ import eu.darken.sdmse.common.shizuku.service.runModuleAction
 import eu.darken.sdmse.common.storage.StorageEnvironment
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.flow.Flow
-import kotlinx.coroutines.flow.onEach
 import kotlinx.coroutines.plus
 import kotlinx.coroutines.withContext
-import okio.*
+import okio.Sink
+import okio.Source
+import okio.sink
+import okio.source
 import java.io.File
 import java.io.IOException
 import java.time.Instant
-import java.util.*
 import javax.inject.Inject
 import javax.inject.Singleton
 
@@ -409,7 +418,7 @@ class LocalGateway @Inject constructor(
                 }
 
                 canRead && mode == Mode.AUTO -> {
-                    log(TAG, VERBOSE) { "walk($mode->NORMAL, direct): $path" }
+                    log(TAG, VERBOSE) { "walk($mode->NORMAL, escalating): $path" }
                     EscalatingWalker(
                         gateway = this@LocalGateway,
                         start = path,
@@ -453,15 +462,77 @@ class LocalGateway @Inject constructor(
 
                 else -> throw IOException("No matching mode available.")
             }
-                .onEach {
-//                    if (Bugs.isTrace) log(TAG, VERBOSE) { "Walked $it" }
-                }
         } catch (e: IOException) {
             log(TAG, WARN) { "walk(path=$path, mode=$mode) failed:\n${e.asLog()}" }
             throw ReadException(path, cause = e)
         }
     }
 
+    override suspend fun du(
+        path: LocalPath,
+        options: APathGateway.DuOptions<LocalPath, LocalPathLookup>,
+    ): Long = du(path, options, Mode.AUTO)
+
+    suspend fun du(
+        path: LocalPath,
+        options: APathGateway.DuOptions<LocalPath, LocalPathLookup> = APathGateway.DuOptions(),
+        mode: Mode = Mode.AUTO,
+    ): Long = runIO {
+        try {
+            val javaFile = path.asFile()
+            val canRead = when (mode) {
+                Mode.AUTO, Mode.NORMAL -> if (javaFile.canRead()) {
+                    try {
+                        javaFile.length()
+                        true
+                    } catch (e: IOException) {
+                        false
+                    }
+                } else {
+                    false
+                }
+
+                else -> false
+            }
+
+            when {
+                mode == Mode.NORMAL -> {
+                    log(TAG, VERBOSE) { "walk($mode->NORMAL, direct): $path" }
+                    if (!canRead) throw ReadException(path)
+                    javaFile.walkTopDown().map { it.length() }.sum()
+                }
+
+                canRead && mode == Mode.AUTO -> {
+                    log(TAG, VERBOSE) { "du($mode->AUTO, escalating): $path" }
+                    try {
+                        du(path, mode = Mode.NORMAL)
+                    } catch (e: ReadException) {
+                        when {
+                            hasRoot() -> du(path, mode = Mode.ROOT)
+                            hasShizuku() -> du(path, mode = Mode.ADB)
+                            else -> throw e
+                        }
+                    }
+                }
+
+                hasRoot() && (mode == Mode.ROOT || !canRead && mode == Mode.AUTO) -> {
+                    log(TAG, VERBOSE) { "du($mode->ROOT): $path" }
+                    rootOps { it.du(path) }
+                }
+
+                hasShizuku() && (mode == Mode.ADB || !canRead && mode == Mode.AUTO) -> {
+                    log(TAG, VERBOSE) { "du($mode->ADB): $path" }
+                    adbOps { it.du(path) }
+                }
+
+                else -> throw IOException("No matching mode available.")
+            }
+        } catch (e: IOException) {
+            log(TAG, WARN) { "du(path=$path, mode=$mode) failed:\n${e.asLog()}" }
+            throw ReadException(path, cause = e)
+        }
+    }
+
     override suspend fun exists(path: LocalPath): Boolean = exists(path, Mode.AUTO)
 
     suspend fun exists(path: LocalPath, mode: Mode = Mode.AUTO): Boolean = runIO {

diff --git a/app-common-io/src/main/java/eu/darken/sdmse/common/files/local/LocalPathExtensions.kt b/app-common-io/src/main/java/eu/darken/sdmse/common/files/local/LocalPathExtensions.kt
@@ -45,7 +45,6 @@ fun LocalPath.performLookup(): LocalPathLookup {
         lookedUp = this,
         size = file.length(),
         modifiedAt = Instant.ofEpochMilli(file.lastModified()),
-
         target = file.readLink()?.let { LocalPath.build(it) }
     )
 }

diff --git a/app-common-io/src/main/java/eu/darken/sdmse/common/files/local/ipc/FileOpsClient.kt b/app-common-io/src/main/java/eu/darken/sdmse/common/files/local/ipc/FileOpsClient.kt
@@ -91,6 +91,12 @@ class FileOpsClient @AssistedInject constructor(
         return output.toLocalPathLookupFlow()
     }
 
+    fun du(path: LocalPath): Long = try {
+        fileOpsConnection.du(path)
+    } catch (e: Exception) {
+        throw e.toFakeIOException()
+    }
+
     fun readFile(path: LocalPath): Source = try {
         fileOpsConnection.readFile(path).source()
     } catch (e: Exception) {

diff --git a/app-common-io/src/main/java/eu/darken/sdmse/common/files/local/ipc/FileOpsHost.kt b/app-common-io/src/main/java/eu/darken/sdmse/common/files/local/ipc/FileOpsHost.kt
@@ -44,7 +44,7 @@ class FileOpsHost @Inject constructor(
     private val ipcFunnel: IPCFunnel,
 ) : FileOpsConnection.Stub(), IpcHostModule {
 
-   private fun listFiles(path: LocalPath): List<LocalPath> = path.asFile().listFiles2().map { LocalPath.build(it) }
+    private fun listFiles(path: LocalPath): List<LocalPath> = path.asFile().listFiles2().map { LocalPath.build(it) }
 
     override fun listFilesStream(path: LocalPath): RemoteInputStream = try {
         if (Bugs.isTrace) log(TAG, VERBOSE) { "listFilesStream($path)..." }
@@ -130,6 +130,14 @@ class FileOpsHost @Inject constructor(
         throw wrapPropagating(e)
     }
 
+    override fun du(path: LocalPath): Long = try {
+        if (Bugs.isTrace) log(TAG, VERBOSE) { "du($path)..." }
+        runBlocking { path.asFile().walkTopDown().map { it.length() }.sum() }
+    } catch (e: Exception) {
+        log(TAG, ERROR) { "exists(path=$path) failed\n${e.asLog()}" }
+        throw wrapPropagating(e)
+    }
+
     override fun readFile(path: LocalPath): RemoteInputStream = try {
         if (Bugs.isTrace) log(TAG, VERBOSE) { "readFile($path)..." }
         FileInputStream(path.asFile()).remoteInputStream()

diff --git a/app-common-io/src/main/java/eu/darken/sdmse/common/files/saf/SAFGateway.kt b/app-common-io/src/main/java/eu/darken/sdmse/common/files/saf/SAFGateway.kt
@@ -323,6 +323,42 @@ class SAFGateway @Inject constructor(
         }
 
 
+    override suspend fun du(
+        path: SAFPath,
+        options: APathGateway.DuOptions<SAFPath, SAFPathLookup>,
+    ): Long = runIO {
+        try {
+            val start = lookup(path)
+            log(TAG, VERBOSE) { "du($path) -> $start" }
+
+            if (start.isFile) return@runIO start.size
+
+            var total = start.size
+
+            val queue = LinkedList(listOf(start))
+            while (!queue.isEmpty()) {
+                val lookUp = queue.removeFirst()
+
+                val newBatch = try {
+                    lookupFiles(lookUp.lookedUp)
+                } catch (e: IOException) {
+                    log(TAG, ERROR) { "Failed to read $lookUp: $e" }
+                    emptyList()
+                }
+
+                newBatch.forEach { child ->
+                    total += child.size
+                    if (child.isDirectory) queue.addFirst(child)
+                }
+            }
+
+            total
+        } catch (e: Exception) {
+            log(TAG, WARN) { "du($path) failed." }
+            throw ReadException(path, cause = e)
+        }
+    }
+
     override suspend fun read(path: SAFPath): Source = runIO {
         try {
             val docFile = findDocFile(path)

diff --git a/app/src/main/java/eu/darken/sdmse/analyzer/core/storage/AppStorageScanner.kt b/app/src/main/java/eu/darken/sdmse/analyzer/core/storage/AppStorageScanner.kt
@@ -180,24 +180,39 @@ class AppStorageScanner @AssistedInject constructor(
         val appMediaGroup = publicMediaPaths.value()
             .map { it.child(request.pkg.packageName) }
             .filter { gatewaySwitch.exists(it, type = GatewaySwitch.Type.AUTO) }
-            .map { it.walkContentItem(gatewaySwitch) }
+            .mapNotNull { path ->
+                try {
+                    if (request.shallow) {
+                        path.sizeContentItem(gatewaySwitch)
+                    } else {
+                        path.walkContentItem(gatewaySwitch)
+                    }
+                } catch (e: ReadException) {
+                    null
+                }
+            }
             .toSet()
             .takeIf { it.isNotEmpty() }
-            ?.let { contentSet ->
+            ?.let {
                 ContentGroup(
                     label = R.string.analyzer_storage_content_app_media_label.toCaString(),
-                    contents = contentSet,
+                    contents = it,
                 )
             }
 
-        val extraData = request.extraData
+        val extraDataGroup = request.extraData
             .mapNotNull { path ->
                 try {
-                    path.walkContentItem(gatewaySwitch)
+                    if (request.shallow) {
+                        path.sizeContentItem(gatewaySwitch)
+                    } else {
+                        path.walkContentItem(gatewaySwitch)
+                    }
                 } catch (e: ReadException) {
                     null
                 }
             }
+            .toSet()
             .takeIf { it.isNotEmpty() }
             ?.let {
                 ContentGroup(
@@ -213,7 +228,7 @@ class AppStorageScanner @AssistedInject constructor(
                 appCode = appCodeGroup,
                 appData = appDataGroup,
                 appMedia = appMediaGroup,
-                extraData = extraData,
+                extraData = extraDataGroup,
             ),
         )
     }