Deduplicator: Don't skip APKs if "skip uncommon" is enabled in settings

Workdoodler · Jul 4, 2024 · 04656d6 · 04656d6
1 parent 964ef0a
commit 04656d6
Show file tree

Hide file tree

Showing 5 changed files with 39 additions and 7 deletions.
diff --git a/app-common-io/src/main/java/eu/darken/sdmse/common/files/APathExtensions.kt b/app-common-io/src/main/java/eu/darken/sdmse/common/files/APathExtensions.kt
@@ -279,4 +279,7 @@ fun Collection<APath>.filterDistinctRoots(): Set<APath> = this
             acc
         }
     }
-    .toSet()
+    .toSet()
+
+val APath.extension: String?
+    get() = name.substringAfterLast('.', "").takeIf { it.isNotEmpty() }
diff --git a/app-common-io/src/main/java/eu/darken/sdmse/common/files/APathLookupExtensions.kt b/app-common-io/src/main/java/eu/darken/sdmse/common/files/APathLookupExtensions.kt
@@ -98,4 +98,7 @@ fun APathLookup<*>.removePrefix(prefix: APath, overlap: Int = 0) =
 fun Collection<APathLookup<*>>.filterDistinctRoots(): Set<APathLookup<*>> {
     val lookupMap = this.associateBy { it.lookedUp }
     return lookupMap.keys.filterDistinctRoots().map { lookupMap.getValue(it) }.toSet()
-}
+}
+
+val APathLookup<*>.extension: String?
+    get() = lookedUp.extension
diff --git a/app-common-io/src/test/java/eu/darken/sdmse/common/files/APathExtensionTest.kt b/app-common-io/src/test/java/eu/darken/sdmse/common/files/APathExtensionTest.kt
@@ -1,6 +1,5 @@
 package eu.darken.sdmse.common.files
 
-import eu.darken.sdmse.common.files.*
 import eu.darken.sdmse.common.files.local.LocalPath
 import eu.darken.sdmse.common.files.local.LocalPathLookup
 import eu.darken.sdmse.common.files.saf.SAFDocFile
@@ -789,4 +788,17 @@ class APathExtensionTest : BaseTest() {
 
         setOf(file1, file1s, file2, file2s).filterDistinctRoots() shouldBe setOf(file1, file2)
     }
+
+
+    @Test fun `file extensions`() {
+        LocalPath.build("test", "file1").extension shouldBe null
+        LocalPath.build("test", "file1.abc").extension shouldBe "abc"
+        LocalPath.build("test", "file1.abc.def").extension shouldBe "def"
+        LocalPath.build("test", "file1.abc..").extension shouldBe null
+
+        SAFPath.build(treeUri, "test", "file2").extension shouldBe null
+        SAFPath.build(treeUri, "test", "file2.abc").extension shouldBe "abc"
+        SAFPath.build(treeUri, "test", "file2.abc.def").extension shouldBe "def"
+        SAFPath.build(treeUri, "test", "file2.abc..").extension shouldBe null
+    }
 }
diff --git a/app/src/main/java/eu/darken/sdmse/deduplicator/core/scanner/CommonFilesCheck.kt b/app/src/main/java/eu/darken/sdmse/deduplicator/core/scanner/CommonFilesCheck.kt
@@ -6,6 +6,7 @@ import eu.darken.sdmse.common.debug.logging.Logging.Priority.VERBOSE
 import eu.darken.sdmse.common.debug.logging.log
 import eu.darken.sdmse.common.debug.logging.logTag
 import eu.darken.sdmse.common.files.APathLookup
+import eu.darken.sdmse.common.files.extension
 import javax.inject.Inject
 
 class CommonFilesCheck @Inject constructor(
@@ -14,17 +15,24 @@ class CommonFilesCheck @Inject constructor(
 
     suspend fun isCommon(lookup: APathLookup<*>): Boolean {
         val mimeType = mimeTypeTool.determineMimeType(lookup)
-        if (Bugs.isTrace) log(TAG, VERBOSE) { "$mimeType <- ${lookup.path}" }
-        return COMMON_TYPES.contains(mimeType)
+        if (Bugs.isDebug) log(TAG, VERBOSE) { "MimeType: $mimeType <- ${lookup.path}" }
+        return when {
+            mimeType == "application/octet-stream" -> APPS_SUFFIXES.contains(lookup.extension)
+            else -> COMMON_TYPES.contains(mimeType)
+        }
     }
 
     suspend fun isImage(lookup: APathLookup<*>): Boolean {
         val mimeType = mimeTypeTool.determineMimeType(lookup)
-        if (Bugs.isTrace) log(TAG, VERBOSE) { "$mimeType <- ${lookup.path}" }
+        if (Bugs.isDebug) log(TAG, VERBOSE) { "MimeType: $mimeType <- ${lookup.path}" }
         return IMAGES.contains(mimeType)
     }
 
     companion object {
+        private val APPS_SUFFIXES = setOf(
+            "apk",
+            "apks"
+        )
         private val IMAGES = setOf(
             "image/x-ms-bmp",
             "image/jpeg",

diff --git a/app/src/main/java/eu/darken/sdmse/deduplicator/core/scanner/checksum/ChecksumSleuth.kt b/app/src/main/java/eu/darken/sdmse/deduplicator/core/scanner/checksum/ChecksumSleuth.kt
@@ -13,6 +13,7 @@ import eu.darken.sdmse.common.areas.currentAreas
 import eu.darken.sdmse.common.ca.toCaString
 import eu.darken.sdmse.common.coroutine.DispatcherProvider
 import eu.darken.sdmse.common.datastore.value
+import eu.darken.sdmse.common.debug.Bugs
 import eu.darken.sdmse.common.debug.logging.Logging.Priority.ERROR
 import eu.darken.sdmse.common.debug.logging.Logging.Priority.VERBOSE
 import eu.darken.sdmse.common.debug.logging.Logging.Priority.WARN
@@ -110,6 +111,7 @@ class ChecksumSleuth @Inject constructor(
 
         val minSize = settings.minSizeBytes.value()
         val skipUncommon = settings.skipUncommon.value()
+        log(TAG) { "Search with minSize=$minSize and skipUncommon=$skipUncommon" }
 
         val suspects = mutableSetOf<APathLookup<*>>()
 
@@ -143,7 +145,11 @@ class ChecksumSleuth @Inject constructor(
                 }
                 .buffer(1024)
                 .filter {
-                    it.isFile && it.size >= minSize && (!skipUncommon || commonFilesCheck.isCommon(it))
+                    if (!it.isFile) return@filter false
+                    val isGoodSize = it.size >= minSize
+                    val isGoodType = (!skipUncommon || commonFilesCheck.isCommon(it))
+                    if (Bugs.isDebug) log(TAG) { "goodSize=$isGoodSize, goodType=$isGoodType <-> $it" }
+                    isGoodSize && isGoodType
                 }
                 .collect { item -> suspects.add(item) }
         }