From ec0cc82b58e255edb9e6c8e79e555dbd405db4b5 Mon Sep 17 00:00:00 2001 From: 770grappenmaker <770grappenmaker@gmail.com> Date: Tue, 30 Jul 2024 19:29:32 +0200 Subject: [PATCH] Add mappings compacting and compression --- app/build.gradle.kts | 3 +- gradle/libs.versions.toml | 3 + mappings/build.gradle.kts | 17 +++ mappings/mappings.versions.toml | 7 ++ mappings/settings.gradle.kts | 13 ++ mappings/src/main/kotlin/mappings.gradle.kts | 125 +++++++++++++++++++ settings.gradle.kts | 1 + 7 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 mappings/build.gradle.kts create mode 100644 mappings/mappings.versions.toml create mode 100644 mappings/settings.gradle.kts create mode 100644 mappings/src/main/kotlin/mappings.gradle.kts diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 817a13c9b5..e5a5210c09 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -3,6 +3,7 @@ plugins { alias(libs.plugins.kotlin.android) alias(libs.plugins.kotlin.compose) alias(libs.plugins.ksp) + id("mappings") } android { @@ -160,4 +161,4 @@ dependencies { detektPlugins(libs.detekt.compose) detektPlugins(libs.detekt.formatting) -} +} \ No newline at end of file diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 9c703ab219..9644e7d2e8 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -29,6 +29,7 @@ core_ktx = { module = "androidx.core:core-ktx", version = "1.13.1" } kotlin_coroutines = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-core", version = "1.8.1" } kotlin_datetime = { module = "org.jetbrains.kotlinx:kotlinx-datetime", version = "0.6.0" } kotlin_immutable = { module = "org.jetbrains.kotlinx:kotlinx-collections-immutable", version = "0.3.7" } +kotlin_serialization_json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version = "1.7.1" } compose_bom = { module = "androidx.compose:compose-bom", version = "2024.06.00" } compose_animation = { module = "androidx.compose.animation:animation" } @@ -77,3 +78,5 @@ desugaring = { module = "com.android.tools:desugar_jdk_libs", version = "2.0.4" detekt_compose = { module = "io.nlopez.compose.rules:detekt", version = "0.4.5" } detekt_formatting = { module = "io.gitlab.arturbosch.detekt:detekt-formatting", version.ref = "detekt" } + +agp = { module = "com.android.tools.build:gradle", version.ref = "agp" } \ No newline at end of file diff --git a/mappings/build.gradle.kts b/mappings/build.gradle.kts new file mode 100644 index 0000000000..e2c9ebc390 --- /dev/null +++ b/mappings/build.gradle.kts @@ -0,0 +1,17 @@ +plugins { + `kotlin-dsl` + alias(rootLibs.plugins.kotlin.serialization) +} + +repositories { + mavenCentral() + google() +} + +dependencies { + implementation(libs.smali) + implementation(libs.smali.dexlib2) + implementation(libs.mappings) + implementation(rootLibs.agp) + implementation(rootLibs.kotlin.serialization.json) +} \ No newline at end of file diff --git a/mappings/mappings.versions.toml b/mappings/mappings.versions.toml new file mode 100644 index 0000000000..a39308fc95 --- /dev/null +++ b/mappings/mappings.versions.toml @@ -0,0 +1,7 @@ +[versions] +smali = "3.0.7" + +[libraries] +smali = { module = "com.android.tools.smali:smali", version.ref = "smali" } +smali_dexlib2 = { module = "com.android.tools.smali:smali-dexlib2", version.ref = "smali" } +mappings = { module = "io.github.770grappenmaker:mappings-util", version = "0.1.7" } \ No newline at end of file diff --git a/mappings/settings.gradle.kts b/mappings/settings.gradle.kts new file mode 100644 index 0000000000..b2783bbdef --- /dev/null +++ b/mappings/settings.gradle.kts @@ -0,0 +1,13 @@ +dependencyResolutionManagement { + versionCatalogs { + create("libs") { + from(files("mappings.versions.toml")) + } + + create("rootLibs") { + from(files("../gradle/libs.versions.toml")) + } + } +} + +rootProject.name = "mappings" \ No newline at end of file diff --git a/mappings/src/main/kotlin/mappings.gradle.kts b/mappings/src/main/kotlin/mappings.gradle.kts new file mode 100644 index 0000000000..275eb028b9 --- /dev/null +++ b/mappings/src/main/kotlin/mappings.gradle.kts @@ -0,0 +1,125 @@ +import com.android.build.gradle.internal.tasks.R8Task +import com.android.tools.smali.dexlib2.DexFileFactory +import com.android.tools.smali.dexlib2.dexbacked.DexBackedDexFile +import com.grappenmaker.mappings.* +import kotlinx.serialization.json.* +import java.io.IOException + +plugins { + id("com.android.application") +} + +private class DexBackedInheritanceProvider(file: DexBackedDexFile) : InheritanceProvider { + private fun String.dropDescriptor() = substring(1, length - 1) + private val classesByName = file.classes.associateBy { it.type.dropDescriptor() } + + override fun getDeclaredMethods(internalName: String, inheritable: Boolean): Iterable { + return (classesByName[internalName] ?: return emptyList()).methods.asSequence() + .let { if (inheritable) it.filter { m -> m.accessFlags and INHERITABLE_MASK == 0 } else it } + .map { "${it.name}(${it.parameterTypes.joinToString("")})${it.returnType}" }.asIterable() + } + + override fun getDirectParents(internalName: String): Iterable { + val def = classesByName[internalName] ?: return emptyList() + + return sequence { + def.superclass?.let { yield(it.dropDescriptor()) } + yieldAll(def.interfaces.asSequence().map { it.dropDescriptor() }) + }.asIterable() + } +} + +abstract class CreateCompactedMappings @Inject constructor() : DefaultTask() { + @get:InputDirectory + abstract val outputDex: DirectoryProperty + + @get:InputFile + abstract val mappingsFile: RegularFileProperty + + @get:OutputFile + abstract val outputFile: RegularFileProperty + + @TaskAction + fun create() { + val dexFilePath = outputDex.get().asFile.walk().drop(1).single() + val dexFile = DexFileFactory.loadDexFile(dexFilePath, null) + val mappingsFilePath = mappingsFile.get().asFile + val mappings = mappingsFilePath.useLines { ProguardMappingsFormat.parse(it) } + val provider = DexBackedInheritanceProvider(dexFile) + + mappings + .reorderNamespaces("official", "named") + .filterClasses { !it.names.first().startsWith("R8\$\$REMOVED\$\$CLASS\$\$") } + .restoreResidualSignatures() + .removeUselessAccessors() + .removeComments() + .removeRedundancy(provider, removeDuplicateMethods = true) + .asCompactedMappings() + .writeTo(outputFile.asFile.get().outputStream()) + } + + private fun Mappings.removeUselessAccessors() = mapClasses { it.removeUselessAccessors() } + + private fun MappedClass.removeUselessAccessors(): MappedClass { + if (methods.size < 2) return this + val deobfedSignatures = methods.mapTo(hashSetOf()) { it.names.last() + it.desc } + + return filterMethods { method -> + val name = method.names.last() + name.length < 8 || // not possibly named accessor + !name.startsWith("access\$") || // not accessor + name[7].isDigit() || // not static accessor (JvmStatic) + (name.drop(8) + method.desc) !in deobfedSignatures // not same signature human-readable + } + } + + private fun MappedMethod.restoreResidualSignature(): MappedMethod { + if (comments.isEmpty()) return this + + val attributes = comments.mapNotNull { runCatching { Json.parseToJsonElement(it).jsonObject }.getOrNull() } + val target = attributes.find { + it["id"]?.jsonPrimitive?.contentOrNull == "com.android.tools.r8.residualsignature" + } ?: return this + + val signature = target["signature"]?.jsonPrimitive?.contentOrNull ?: return this + return if (desc != signature) copy(desc = signature) else this + } + + private fun Mappings.restoreResidualSignatures(): Mappings = mapMethods { _, m -> m.restoreResidualSignature() } +} + +tasks { + afterEvaluate { + val minifyReleaseWithR8 by getting(R8Task::class) + val createCompactedMappings by registering(CreateCompactedMappings::class) { + group = "compacted mappings" + dependsOn(minifyReleaseWithR8) + + outputDex = minifyReleaseWithR8.outputDex + mappingsFile = minifyReleaseWithR8.mappingFile + outputFile = layout.buildDirectory.get().dir("outputs").dir("mapping").file("mapping.compact") + } + + val compressCompactedMappings by registering { + group = "compacted mappings" + + dependsOn(createCompactedMappings) + val uncompressedFile = createCompactedMappings.get().outputs.files.singleFile.absolutePath + + doLast { + // Using try-catch to allow failure, when xz is not on path + // TODO: do we force xz to be on path? + // Using ProcessBuilder such that the task is cacheable + // exec {} has a receiver on Project which is never cacheable + try { + ProcessBuilder("xz", "-k", "-9", "-f", "-e", uncompressedFile) + .redirectError(ProcessBuilder.Redirect.INHERIT) + .start().waitFor() + } catch (e: IOException) { + logger.warn("Could not compress mappings, XZ might not be on PATH") + logger.trace("Compressing mappings failed", e) + } + } + } + } +} \ No newline at end of file diff --git a/settings.gradle.kts b/settings.gradle.kts index 8a68f816b5..2c752fad41 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -40,3 +40,4 @@ include(":providers:kugou") include(":providers:lrclib") include(":providers:piped") include(":providers:translate") +includeBuild("mappings") \ No newline at end of file