Fix percent processing (again)

adriangl · adriangl · commit 6ef22c85588b · 2020-12-28T18:14:17.000+01:00
diff --git a/.gitignore b/.gitignore
@@ -124,4 +124,7 @@ gradle-app.setting
 ### Gradle Patch ###
 **/build/
 
-# End of https://www.gitignore.io/api/java,gradle,intellij+iml
+# End of https://www.gitignore.io/api/java,gradle,intellij+iml
+
+# Output directory for tests
+/output
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -20,19 +20,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - No security issues fixed!
 -->
 
-## [Unreleased]
-### Added
-- No new features!
-### Changed
-- No changed features!
-### Deprecated
-- No deprecated features!
-### Removed
-- No removed features!
+## [1.4.1] - 2020-12-28
 ### Fixed
-- No fixed issues!
-### Security
-- No security issues fixed!
+- Fix percent symbols not being properly escaped (again) by processing the XML file line by line.
 
 ## [1.4.0] - 2020-12-25
 ### Added
@@ -233,7 +223,8 @@ res_dir_path -> resDirPath
 ### Added
 - Initial release.
 
-[Unreleased]: https://github.com/bq/poeditor-android-gradle-plugin/compare/1.4.0...HEAD
+[Unreleased]: https://github.com/bq/poeditor-android-gradle-plugin/compare/1.4.1...HEAD
+[1.4.1]: https://github.com/bq/poeditor-android-gradle-plugin/compare/1.4.0...1.4.1
 [1.4.0]: https://github.com/bq/poeditor-android-gradle-plugin/compare/1.3.1...1.4.0
 [1.3.1]: https://github.com/bq/poeditor-android-gradle-plugin/compare/1.3.0...1.3.1
 [1.3.0]: https://github.com/bq/poeditor-android-gradle-plugin/compare/1.2.0...1.3.0
diff --git a/README.md b/README.md
@@ -18,7 +18,7 @@ buildscript {
         maven { url 'https://jitpack.io' }
     }
     dependencies {
-        classpath 'com.github.bq:poeditor-android-gradle-plugin:1.4.0'
+        classpath 'com.github.bq:poeditor-android-gradle-plugin:1.4.1'
     }
 }
 ```
@@ -33,7 +33,7 @@ buildscript {
         maven("https://jitpack.io")
     }
     dependencies {
-        classpath("com.github.bq:poeditor-android-gradle-plugin:1.4.0")
+        classpath("com.github.bq:poeditor-android-gradle-plugin:1.4.1")
     }
 }
 ```
diff --git a/src/main/kotlin/com/bq/poeditor/gradle/Main.kt b/src/main/kotlin/com/bq/poeditor/gradle/Main.kt
@@ -23,7 +23,7 @@ package com.bq.poeditor.gradle
 fun main() {
     val apiToken = "your_api_token"
     val projectId = 1234567890
-    val resDirPath = "your_res_dir_path"
+    val resDirPath = "output"
     val defaultLanguage = "en"
 
     PoEditorStringsImporter.importPoEditorStrings(apiToken, projectId, defaultLanguage, resDirPath)
diff --git a/src/main/kotlin/com/bq/poeditor/gradle/ktx/DocumentExtensions.kt b/src/main/kotlin/com/bq/poeditor/gradle/ktx/DocumentExtensions.kt
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2020 BQ
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.bq.poeditor.gradle.ktx
+
+import org.w3c.dom.Document
+import org.w3c.dom.bootstrap.DOMImplementationRegistry
+import org.w3c.dom.ls.DOMImplementationLS
+import java.io.StringWriter
+import java.lang.Boolean
+import javax.xml.parsers.DocumentBuilderFactory
+
+private val DEFAULT_ENCODING = Charsets.UTF_8
+
+/**
+ * Converts an XML string to a proper [Document].
+ */
+fun String.toDocument(): Document =
+    DocumentBuilderFactory.newInstance()
+        .newDocumentBuilder()
+        .parse(this.byteInputStream(DEFAULT_ENCODING))
+
+/**
+ * Convers a [Document] into a formatted [String].
+ */
+fun Document.dumpToString(): String {
+    val registry = DOMImplementationRegistry.newInstance()
+    val impl = registry.getDOMImplementation("LS") as DOMImplementationLS
+    val output = impl.createLSOutput().apply { encoding = "UTF-8" }
+    val serializer = impl.createLSSerializer()
+
+    val writer = StringWriter()
+    output.characterStream = writer
+
+    serializer.domConfig.setParameter("format-pretty-print",
+        Boolean.TRUE)
+    serializer.domConfig.setParameter("xml-declaration", true)
+
+    serializer.write(this, output)
+
+    return writer.toString()
+}
diff --git a/src/main/kotlin/com/bq/poeditor/gradle/xml/AndroidXmlWriter.kt b/src/main/kotlin/com/bq/poeditor/gradle/xml/AndroidXmlWriter.kt
@@ -16,15 +16,13 @@
 
 package com.bq.poeditor.gradle.xml
 
+import com.bq.poeditor.gradle.ktx.dumpToString
 import com.bq.poeditor.gradle.utils.TABLET_REGEX_STRING
 import com.bq.poeditor.gradle.utils.TABLET_RES_FOLDER_SUFFIX
 import com.bq.poeditor.gradle.utils.logger
 import com.bq.poeditor.gradle.utils.createValuesModifierFromLangCode
 import org.w3c.dom.Document
-import org.w3c.dom.bootstrap.DOMImplementationRegistry
-import org.w3c.dom.ls.DOMImplementationLS
 import java.io.File
-import java.io.StringWriter
 import java.lang.IllegalStateException
 
 /**
@@ -70,19 +68,6 @@ class AndroidXmlWriter {
             }
         }
 
-        val registry = DOMImplementationRegistry.newInstance()
-        val impl = registry.getDOMImplementation("LS") as DOMImplementationLS
-        val output = impl.createLSOutput().apply { encoding = "UTF-8" }
-        val serializer = impl.createLSSerializer()
-
-        val writer = StringWriter()
-        output.characterStream = writer
-
-        serializer.domConfig.setParameter("format-pretty-print",
-            java.lang.Boolean.TRUE)
-        serializer.domConfig.setParameter("xml-declaration", true)
-
-        serializer.write(document, output)
-        File(stringsFolderFile, "strings.xml").writeText(writer.toString())
+        File(stringsFolderFile, "strings.xml").writeText(document.dumpToString())
     }
 }
diff --git a/src/main/kotlin/com/bq/poeditor/gradle/xml/XmlPostProcessor.kt b/src/main/kotlin/com/bq/poeditor/gradle/xml/XmlPostProcessor.kt
@@ -16,6 +16,8 @@
 
 package com.bq.poeditor.gradle.xml
 
+import com.bq.poeditor.gradle.ktx.dumpToString
+import com.bq.poeditor.gradle.ktx.toDocument
 import com.bq.poeditor.gradle.utils.ALL_REGEX_STRING
 import org.w3c.dom.Document
 import org.w3c.dom.Element
@@ -31,6 +33,13 @@ class XmlPostProcessor {
     companion object {
         private val DEFAULT_ENCODING = Charsets.UTF_8
         private val VARIABLE_REGEX = Regex("""\{\d?\{(.*?)\}\}""")
+
+        private const val TAG_RESOURCES = "resources"
+        private const val TAG_STRING = "string"
+        private const val TAG_PLURALS = "plurals"
+        private const val TAG_ITEM = "item"
+
+        private const val ATTR_NAME = "name"
     }
 
     /**
@@ -40,17 +49,29 @@ class XmlPostProcessor {
      * - Format variables and texts to conform to Android strings.xml format
      * - Split to multiple XML files depending on regex matching
      */
-    fun postProcessTranslationXml(translationXmlString: String,
+    fun postProcessTranslationXml(translationFileXmlString: String,
                                   fileSplitRegexStringList: List<String>): Map<String, Document> =
-        splitTranslationXml(formatTranslationXml(translationXmlString), fileSplitRegexStringList)
+        splitTranslationXml(formatTranslationXml(translationFileXmlString), fileSplitRegexStringList)
+
+    /**
+     * Formats a given translations XML string to conform to Android strings.xml format.
+     */
+    fun formatTranslationXml(translationFileXmlString: String): String {
+        // Parse line by line by traversing the original file using DOM
+        val translationFileXmlDocument = translationFileXmlString.toDocument()
+
+        formatTranslationXmlDocument(translationFileXmlDocument, translationFileXmlDocument.childNodes)
+
+        return translationFileXmlDocument.dumpToString()
+    }
 
     /**
-     * Format variables and texts to conform to Android strings.xml format.
+     * Formats a given string to conform to Android strings.xml format.
      */
-    fun formatTranslationXml(translationXmlString: String): String {
-        // We need to check for variables to see if we have to escape percent symbols: if we find variables, we have to
-        // escape them
-        val containsVariables = translationXmlString.contains(VARIABLE_REGEX)
+    fun formatTranslationString(translationString: String): String {
+        // We need to check for variables to see if we have to escape percent symbols: if we find variables, we have
+        // to escape them
+        val containsVariables = translationString.contains(VARIABLE_REGEX)
 
         val placeholderTransform: (MatchResult) -> CharSequence = { matchResult ->
             // TODO: if the string has multiple variables but any of them has no order number,
@@ -64,7 +85,7 @@ class XmlPostProcessor {
             }
         }
 
-        return translationXmlString
+        return translationString
             // Replace % with %% if variables are found
             .let { if (containsVariables) it.replace("%", "%%") else it }
             // Replace &lt; with < and &gt; with >
@@ -98,9 +119,9 @@ class XmlPostProcessor {
                 nodes.forEach { node ->
                     node.parentNode.removeChild(node)
                     val copiedNode = (node.cloneNode(true) as Element).apply {
-                        val name = getAttribute("name")
+                        val name = getAttribute(ATTR_NAME)
                         val nameWithoutRegex = regex.find(name)?.groups?.get(1)?.value ?: ""
-                        setAttribute("name", nameWithoutRegex)
+                        setAttribute(ATTR_NAME, nameWithoutRegex)
                     }
                     xmlRecords.adoptNode(copiedNode)
                     xmlRecords.firstChild.appendChild(copiedNode)
@@ -113,25 +134,65 @@ class XmlPostProcessor {
             .plus(ALL_REGEX_STRING to translationFileRecords)
     }
 
+    private fun formatTranslationXmlDocument(document: Document, nodeList: NodeList, rootNode: Node? = null) {
+        for (i in 0 until nodeList.length) {
+            if (nodeList.item(i).nodeType == Node.ELEMENT_NODE) {
+                val nodeElement = nodeList.item(i) as Element
+                when (nodeElement.tagName) {
+                    TAG_RESOURCES -> {
+                        // Main node, traverse its children
+                        formatTranslationXmlDocument(document, nodeElement.childNodes, nodeElement)
+                    }
+                    TAG_PLURALS -> {
+                        // Plurals node, process its children
+                        formatTranslationXmlDocument(document, nodeElement.childNodes, nodeElement)
+                    }
+                    TAG_STRING -> {
+                        // String node, apply transformation to the content
+                        processTextAndReplaceNodeContent(document, nodeElement, rootNode)
+                    }
+                    TAG_ITEM -> {
+                        // Plurals item node, apply transformation to the content
+                        processTextAndReplaceNodeContent(document, nodeElement, rootNode)
+                    }
+                }
+            }
+        }
+    }
+
+    private fun processTextAndReplaceNodeContent(document: Document, nodeElement: Element, rootNode: Node?) {
+        val content = nodeElement.textContent
+        val processedContent = formatTranslationString(content)
+        val copiedNodeElement = (nodeElement.cloneNode(true) as Element).apply {
+            textContent = processedContent
+        }
+        document.adoptNode(copiedNodeElement)
+        rootNode?.replaceChild(copiedNodeElement, nodeElement)
+    }
+
     private fun extractMatchingNodes(nodeList: NodeList, regexString: String): List<Node> {
         val matchedNodes = mutableListOf<Node>()
         val regex = Regex(regexString)
 
         for (i in 0 until nodeList.length) {
             if (nodeList.item(i).nodeType == Node.ELEMENT_NODE) {
                 val nodeElement = nodeList.item(i) as Element
-                when {
-                    // Main XML node, process children
-                    nodeElement.tagName == "resources" -> {
+                when (nodeElement.tagName) {
+                    TAG_RESOURCES -> {
+                        // Main XML node, process children
                         matchedNodes.addAll(extractMatchingNodes(nodeElement.childNodes, regexString))
                     }
-                    // String node, add if name matches regex
-                    nodeElement.tagName == "string" && nodeElement.getAttribute("name").matches(regex) -> {
-                        matchedNodes.add(nodeElement)
+                    TAG_STRING -> {
+                        // String node, add node if name matches regex
+                        if (nodeElement.getAttribute(ATTR_NAME).matches(regex)) {
+                            matchedNodes.add(nodeElement)
+                        }
                     }
-                    // Plurals node, add node and children if name matches regex
-                    nodeElement.tagName == "plurals" && nodeElement.getAttribute("name").matches(regex) -> {
-                        matchedNodes.add(nodeElement)
+                    TAG_PLURALS -> {
+                        // Plurals node, add node and children if name matches regex
+                        if (nodeElement.getAttribute(ATTR_NAME).matches(regex)) {
+                            matchedNodes.add(nodeElement)
+                        }
                     }
                 }
             }
diff --git a/src/test/kotlin/com/bq/poeditor/gradle/xml/XmlPostProcessorTest.kt b/src/test/kotlin/com/bq/poeditor/gradle/xml/XmlPostProcessorTest.kt

Original file line number	Diff line number	Diff line change
`@@ -18,7 +18,7 @@ buildscript {`
`18`	`18`	`maven { url 'https://jitpack.io' }`
`19`	`19`	`}`
`20`	`20`	`dependencies {`
`21`		`- classpath 'com.github.bq:poeditor-android-gradle-plugin:1.4.0'`
	`21`	`+ classpath 'com.github.bq:poeditor-android-gradle-plugin:1.4.1'`
`22`	`22`	`}`
`23`	`23`	`}`
`24`	`24`	```
`@@ -33,7 +33,7 @@ buildscript {`
`33`	`33`	`maven("https://jitpack.io")`
`34`	`34`	`}`
`35`	`35`	`dependencies {`
`36`		`- classpath("com.github.bq:poeditor-android-gradle-plugin:1.4.0")`
	`36`	`+ classpath("com.github.bq:poeditor-android-gradle-plugin:1.4.1")`
`37`	`37`	`}`
`38`	`38`	`}`
`39`	`39`	```