Rewrite the XML parsing logic (#78)

Author: adriangl

.gitignore

@@ -13,6 +13,7 @@
 .idea/**/dictionaries
 .idea/**/shelf
 .idea/**/runConfigurations.xml
+.idea/**/other.xml
 .idea/kotlinc.xml
 
 # Generated files

.idea/migrations.xml

@@ -24,7 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - No new features!
 ### Changed
-- No changed features!
+- Refactor the whole parsing logic to better handle input and improve performance.
 ### Deprecated
 - No deprecated features!
 ### Removed

CHANGELOG.md

@@ -50,11 +50,13 @@ dependencies {
 
     compileOnly(libs.android.buildTools)
 
+
     implementation(libs.kotlin.stdlib)
     implementation(libs.bundles.moshi)
     implementation(libs.bundles.retrofit)
     implementation(libs.bundles.okhttp3)
     implementation(libs.dotenvKotlin)
+    implementation(libs.xerces)
 
     testImplementation(gradleTestKit())
     testImplementation(kotlin("test"))

build.gradle.kts

@@ -13,25 +13,27 @@ android-buildTools = "com.android.tools.build:gradle:8.1.2"
 kotlin-gradle = { module = "org.jetbrains.kotlin:kotlin-gradle-plugin", version.ref = "kotlin" }
 kotlin-stdlib = { module = "org.jetbrains.kotlin:kotlin-stdlib", version.ref = "kotlin" }
 
-moshi = { group = "com.squareup.moshi", name = "moshi", version.ref = "moshi"}
-moshi-kotlin = { group = "com.squareup.moshi", name = "moshi-kotlin", version.ref = "moshi"}
-moshi-adapters = { group = "com.squareup.moshi", name = "moshi-adapters", version.ref = "moshi"}
+moshi = { group = "com.squareup.moshi", name = "moshi", version.ref = "moshi" }
+moshi-kotlin = { group = "com.squareup.moshi", name = "moshi-kotlin", version.ref = "moshi" }
+moshi-adapters = { group = "com.squareup.moshi", name = "moshi-adapters", version.ref = "moshi" }
 
-retrofit = { group = "com.squareup.retrofit2", name = "retrofit", version.ref = "retrofit"}
-retrofit-converterMoshi = { group = "com.squareup.retrofit2", name = "converter-moshi", version.ref = "retrofit"}
+retrofit = { group = "com.squareup.retrofit2", name = "retrofit", version.ref = "retrofit" }
+retrofit-converterMoshi = { group = "com.squareup.retrofit2", name = "converter-moshi", version.ref = "retrofit" }
 
-okhttp3 = { group = "com.squareup.okhttp3", name = "okhttp", version.ref = "okhttp"}
-okhttp3-loggingInterceptor = { group = "com.squareup.okhttp3", name = "logging-interceptor", version.ref = "okhttp"}
+okhttp3 = { group = "com.squareup.okhttp3", name = "okhttp", version.ref = "okhttp" }
+okhttp3-loggingInterceptor = { group = "com.squareup.okhttp3", name = "logging-interceptor", version.ref = "okhttp" }
 
 dotenvKotlin = "io.github.cdimascio:dotenv-kotlin:6.4.1"
 
+xerces = { module = "xerces:xercesImpl", version = "2.12.2" }
+
 junit = "junit:junit:4.13.2"
 
-detekt-formatting = { group = "io.gitlab.arturbosch.detekt", name = "detekt-formatting", version.ref = "detekt"}
+detekt-formatting = { group = "io.gitlab.arturbosch.detekt", name = "detekt-formatting", version.ref = "detekt" }
 
 [plugins]
 detekt = { id = "io.gitlab.arturbosch.detekt", version.ref = "detekt" }
-gitVersionGradle = { id = "com.gladed.androidgitversion", version = "0.4.14"}
+gitVersionGradle = { id = "com.gladed.androidgitversion", version = "0.4.14" }
 versionsUpdate = { id = "com.github.ben-manes.versions", version = "0.49.0" }
 
 [bundles]

gradle/libs.versions.toml

@@ -24,7 +24,7 @@ import com.android.build.gradle.AppPlugin
 import com.android.build.gradle.LibraryExtension
 import com.android.build.gradle.LibraryPlugin
 import com.android.build.gradle.internal.dsl.BaseAppModuleExtension
-import com.hyperdevs.poeditor.gradle.ktx.registerNewTask
+import com.hyperdevs.poeditor.gradle.extensions.registerNewTask
 import com.hyperdevs.poeditor.gradle.tasks.ImportPoEditorStringsTask
 import com.hyperdevs.poeditor.gradle.utils.*
 import org.gradle.api.NamedDomainObjectContainer

src/main/kotlin/com/hyperdevs/poeditor/gradle/PoEditorPlugin.kt

@@ -19,7 +19,7 @@
 package com.hyperdevs.poeditor.gradle
 
 import com.hyperdevs.poeditor.gradle.adapters.PoEditorDateJsonAdapter
-import com.hyperdevs.poeditor.gradle.ktx.downloadUrlToString
+import com.hyperdevs.poeditor.gradle.extensions.downloadUrlToString
 import com.hyperdevs.poeditor.gradle.network.PoEditorApiControllerImpl
 import com.hyperdevs.poeditor.gradle.network.api.ExportType
 import com.hyperdevs.poeditor.gradle.network.api.FilterType
@@ -28,8 +28,9 @@ import com.hyperdevs.poeditor.gradle.network.api.PoEditorApi
 import com.hyperdevs.poeditor.gradle.network.api.ProjectLanguage
 import com.hyperdevs.poeditor.gradle.utils.TABLET_REGEX_STRING
 import com.hyperdevs.poeditor.gradle.utils.logger
-import com.hyperdevs.poeditor.gradle.xml.AndroidXmlWriter
-import com.hyperdevs.poeditor.gradle.xml.XmlPostProcessor
+import com.hyperdevs.poeditor.gradle.xml.StringsXmlPostProcessor
+import com.hyperdevs.poeditor.gradle.xml.StringsXmlWriter
+import com.hyperdevs.poeditor.gradle.xml.parser.SaxStringsXmlDocumentParser
 import com.squareup.moshi.Moshi
 import com.squareup.moshi.kotlin.reflect.KotlinJsonAdapterFactory
 import okhttp3.HttpUrl.Companion.toHttpUrl
@@ -59,12 +60,14 @@ object PoEditorStringsImporter {
         .connectTimeout(CONNECT_TIMEOUT_SECONDS, TimeUnit.SECONDS)
         .readTimeout(READ_TIMEOUT_SECONDS, TimeUnit.SECONDS)
         .writeTimeout(WRITE_TIMEOUT_SECONDS, TimeUnit.SECONDS)
-        .addInterceptor(HttpLoggingInterceptor(object : HttpLoggingInterceptor.Logger {
-            override fun log(message: String) {
-                logger.debug(message)
-            }
-        })
-            .setLevel(HttpLoggingInterceptor.Level.BODY))
+        .addInterceptor(
+            HttpLoggingInterceptor(object : HttpLoggingInterceptor.Logger {
+                override fun log(message: String) {
+                    logger.debug(message)
+                }
+            })
+                .setLevel(HttpLoggingInterceptor.Level.BODY)
+        )
         .build()
 
     private val retrofit = Retrofit.Builder()
@@ -75,27 +78,25 @@ object PoEditorStringsImporter {
 
     private val poEditorApi: PoEditorApi = retrofit.create(PoEditorApi::class.java)
 
-    private val xmlPostProcessor = XmlPostProcessor()
-
-    private val xmlWriter = AndroidXmlWriter()
-
     /**
      * Imports PoEditor strings.
      */
-    @Suppress("LongParameterList")
-    fun importPoEditorStrings(apiToken: String,
-                              projectId: Int,
-                              defaultLang: String,
-                              resDirPath: String,
-                              filters: List<FilterType>,
-                              order: OrderType,
-                              tags: List<String>,
-                              languageValuesOverridePathMap: Map<String, String>,
-                              minimumTranslationPercentage: Int,
-                              resFileName: String,
-                              unquoted: Boolean,
-                              unescapeHtmlTags: Boolean,
-                              untranslatableStringsRegex: String?) {
+    @Suppress("LongParameterList", "LongMethod")
+    fun importPoEditorStrings(
+        apiToken: String,
+        projectId: Int,
+        defaultLang: String,
+        resDirPath: String,
+        filters: List<FilterType>,
+        order: OrderType,
+        tags: List<String>,
+        languageValuesOverridePathMap: Map<String, String>,
+        minimumTranslationPercentage: Int,
+        resFileName: String,
+        unquoted: Boolean,
+        unescapeHtmlTags: Boolean,
+        untranslatableStringsRegex: String?
+    ) {
         try {
             val poEditorApiController = PoEditorApiControllerImpl(apiToken, moshi, poEditorApi)
 
@@ -122,6 +123,8 @@ object PoEditorStringsImporter {
                 logger.lifecycle("Using the following filters for all languages: $filters")
             }
 
+            val stringsXmlDocumentParser = SaxStringsXmlDocumentParser()
+
             projectLanguages.minus(skippedLanguages).forEach { languageData ->
                 val languageCode = languageData.code
 
@@ -143,26 +146,31 @@ object PoEditorStringsImporter {
                 val translationFile = okHttpClient.downloadUrlToString(translationFileUrl)
 
                 // Extract final files from downloaded translation XML
-                val postProcessedXmlDocumentMap = xmlPostProcessor.postProcessTranslationXml(
-                    translationFile,
+                val originalStringsXmlDocument = stringsXmlDocumentParser.deserialize(translationFile)
+
+                val postProcessedStringsXmlDocumentMap = StringsXmlPostProcessor.processTranslationXml(
+                    originalStringsXmlDocument,
                     listOf(TABLET_REGEX_STRING),
                     unescapeHtmlTags,
-                    untranslatableStringsRegex
+                    untranslatableStringsRegex?.toRegex()
                 )
 
-                xmlWriter.saveXml(
+                StringsXmlWriter.saveXml(
                     resDirPath,
                     resFileName,
-                    postProcessedXmlDocumentMap,
+                    postProcessedStringsXmlDocumentMap.mapValues { (_, document) ->
+                        stringsXmlDocumentParser.serialize(document)
+                    },
                     defaultLang,
                     languageCode,
-                    languageValuesOverridePathMap,
-                    unescapeHtmlTags
+                    languageValuesOverridePathMap
                 )
             }
         } catch (e: Exception) {
-            logger.error("An error happened when retrieving strings from project. " +
-                         "Please review the plug-in's input parameters and try again")
+            logger.error(
+                "An error happened when retrieving strings from project. " +
+                "Please review the plug-in's input parameters and try again"
+            )
             throw e
         }
     }

src/main/kotlin/com/hyperdevs/poeditor/gradle/PoEditorStringsImporter.kt

@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package com.hyperdevs.poeditor.gradle.ktx
+package com.hyperdevs.poeditor.gradle.extensions
 
 import okhttp3.OkHttpClient
 import okhttp3.Request

src/main/kotlin/com/hyperdevs/poeditor/gradle/ktx/OkHttpClientExtensions.kt renamed to src/main/kotlin/com/hyperdevs/poeditor/gradle/extensions/OkHttpClientExtensions.kt

@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package com.hyperdevs.poeditor.gradle.ktx
+package com.hyperdevs.poeditor.gradle.extensions
 
 import org.gradle.api.Project
 import org.gradle.api.Task

src/main/kotlin/com/hyperdevs/poeditor/gradle/ktx/ProjectExtensions.kt renamed to src/main/kotlin/com/hyperdevs/poeditor/gradle/extensions/ProjectExtensions.kt

@@ -0,0 +1,56 @@
+/*
+ * Copyright 2021 HyperDevs
+ *
+ * Copyright 2020 BQ
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.hyperdevs.poeditor.gradle.extensions
+
+private val cdataRegex = Regex("^<!\\[CDATA\\[(.*)]]>$", RegexOption.DOT_MATCHES_ALL)
+
+/**
+ * Unescapes HTML tags from string.
+ */
+fun String.unescapeHtmlTags() = this
+    .replace("&lt;", "<")
+    .replace("&gt;", ">")
+    .replace("&amp;", "&")
+    .replace("&apos;", "'")
+    .replace("&quot;", "\"")
+
+/**
+ * Returns if the given string is a CDATA string.
+ */
+fun String.isCData() = cdataRegex.matches(this.trim())
+
+/**
+ * Returns the CDATA content from a given CDATA string.
+ *
+ * Returns null if the string is not a CDATA string.
+ */
+fun String.getCDataContent() = cdataRegex.matchEntire(this.trim())?.groups?.get(1)?.value
+
+/**
+ * Replaces the CDATA contents of a given string.
+ *
+ * Returns the same string if it's not a CDATA string.
+ */
+fun String.replaceCDataContent(newContent: String): String {
+    return if (this.isCData()) {
+        "<![CDATA[$newContent]]>"
+    } else {
+        this
+    }
+}