Moved bloom filter binary to Play flavor (#1120)

* added a httpsdatapersister that does not persist embedded data for froid

* more code cleanup

* we don't need to duplicate so much code

* fixed bloom filter test

* spotless check

* improved logging statements

Author: malmstein

app/src/androidTest/java/com/duckduckgo/app/integration/HttpsEmbeddedDataIntegrationTest.kt

@@ -28,6 +28,7 @@ import com.duckduckgo.app.httpsupgrade.api.HttpsFalsePositivesJsonAdapter
 import com.duckduckgo.app.httpsupgrade.store.HttpsDataPersister
 import com.duckduckgo.app.privacy.db.UserWhitelistDao
 import com.duckduckgo.app.statistics.pixels.Pixel
+import com.duckduckgo.httpsupgrade.store.PlayHttpsEmbeddedDataPersister
 import com.nhaarman.mockitokotlin2.mock
 import com.squareup.moshi.Moshi
 import org.junit.After
@@ -51,20 +52,21 @@ class HttpsEmbeddedDataIntegrationTest {
         db = Room.inMemoryDatabaseBuilder(context, AppDatabase::class.java)
             .allowMainThreadQueries()
             .build()
+
         var httpsBloomSpecDao = db.httpsBloomFilterSpecDao()
         var httpsFalsePositivesDao = db.httpsFalsePositivesDao()
-        var binaryDataStore: BinaryDataStore = BinaryDataStore(context)
+        var binaryDataStore = BinaryDataStore(context)
 
         val persister = HttpsDataPersister(
             binaryDataStore,
             httpsBloomSpecDao,
             httpsFalsePositivesDao,
-            db,
-            context,
-            moshi
+            db
         )
 
-        val factory = HttpsBloomFilterFactoryImpl(httpsBloomSpecDao, binaryDataStore, persister)
+        var embeddedDataPersister = PlayHttpsEmbeddedDataPersister(persister, binaryDataStore, httpsBloomSpecDao, context, moshi)
+
+        val factory = HttpsBloomFilterFactoryImpl(httpsBloomSpecDao, binaryDataStore, embeddedDataPersister, persister)
         httpsUpgrader = HttpsUpgraderImpl(factory, httpsFalsePositivesDao, mockUserAllowlistDao, mockPixel)
         httpsUpgrader.reloadData()
     }

app/src/fdroid/java/com/duckduckgo/app/di/HttpsPersisterModule.kt

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2021 DuckDuckGo
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.duckduckgo.app.di
+
+import com.duckduckgo.app.httpsupgrade.store.HttpsEmbeddedDataPersister
+import com.duckduckgo.httpsupgrade.store.FDroidHttpsEmbeddedDataPersister
+import dagger.Module
+import dagger.Provides
+
+@Module
+class HttpsPersisterModule {
+
+    @Provides
+    fun providesHttpsDataManager(): HttpsEmbeddedDataPersister {
+        return FDroidHttpsEmbeddedDataPersister()
+    }
+
+}

app/src/fdroid/java/com/duckduckgo/httpsupgrade/store/FDroidHttpsEmbeddedDataPersister.kt

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2021 DuckDuckGo
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.duckduckgo.httpsupgrade.store
+
+import com.duckduckgo.app.httpsupgrade.store.HttpsEmbeddedDataPersister
+import timber.log.Timber
+
+class FDroidHttpsEmbeddedDataPersister : HttpsEmbeddedDataPersister {
+
+    override fun shouldPersistEmbeddedData(): Boolean {
+        Timber.d("Ignoring, FDroid does not use embedded data due to binary data restrictions")
+        return false
+    }
+
+    override fun persistEmbeddedData() {
+        Timber.d("Ignoring, FDroid does not use embedded data due to binary data restrictions")
+    }
+}

app/src/main/java/com/duckduckgo/app/di/AppComponent.kt

@@ -78,7 +78,8 @@ import javax.inject.Singleton
         StoreReferralModule::class,
         CoroutinesModule::class,
         CertificateTrustedStoreModule::class,
-        WelcomePageModule::class
+        WelcomePageModule::class,
+        HttpsPersisterModule::class
     ]
 )
 interface AppComponent : AndroidInjector<DuckDuckGoApplication> {

app/src/main/java/com/duckduckgo/app/httpsupgrade/HttpsBloomFilterFactory.kt

@@ -20,6 +20,7 @@ import androidx.annotation.WorkerThread
 import com.duckduckgo.app.global.store.BinaryDataStore
 import com.duckduckgo.app.httpsupgrade.model.HttpsBloomFilterSpec.Companion.HTTPS_BINARY_FILE
 import com.duckduckgo.app.httpsupgrade.store.HttpsBloomFilterSpecDao
+import com.duckduckgo.app.httpsupgrade.store.HttpsEmbeddedDataPersister
 import com.duckduckgo.app.httpsupgrade.store.HttpsDataPersister
 import timber.log.Timber
 import javax.inject.Inject
@@ -31,21 +32,22 @@ interface HttpsBloomFilterFactory {
 class HttpsBloomFilterFactoryImpl @Inject constructor(
     private val dao: HttpsBloomFilterSpecDao,
     private val binaryDataStore: BinaryDataStore,
-    private val persister: HttpsDataPersister
+    private val httpsEmbeddedDataPersister: HttpsEmbeddedDataPersister,
+    private val httpsDataPersister: HttpsDataPersister
 ) : HttpsBloomFilterFactory {
 
     @WorkerThread
     override fun create(): BloomFilter? {
 
-        if (!persister.isPersisted()) {
+        if (httpsEmbeddedDataPersister.shouldPersistEmbeddedData()) {
             Timber.d("Https update data not found, loading embedded data")
-            persister.persistEmbeddedData()
+            httpsEmbeddedDataPersister.persistEmbeddedData()
         }
 
         val specification = dao.get()
         val dataPath = binaryDataStore.dataFilePath(HTTPS_BINARY_FILE)
-        if (dataPath == null || specification == null || !persister.isPersisted(specification)) {
-            Timber.d("Embedded https update data failed to load")
+        if (dataPath == null || specification == null || !httpsDataPersister.isPersisted(specification)) {
+            Timber.d("Https update data not available")
             return null
         }
 

app/src/main/java/com/duckduckgo/app/httpsupgrade/di/HttpsUpgraderModule.kt

@@ -23,6 +23,7 @@ import com.duckduckgo.app.httpsupgrade.HttpsBloomFilterFactory
 import com.duckduckgo.app.httpsupgrade.HttpsBloomFilterFactoryImpl
 import com.duckduckgo.app.httpsupgrade.store.HttpsBloomFilterSpecDao
 import com.duckduckgo.app.httpsupgrade.store.HttpsDataPersister
+import com.duckduckgo.app.httpsupgrade.store.HttpsEmbeddedDataPersister
 import com.duckduckgo.app.httpsupgrade.store.HttpsFalsePositivesDao
 import com.duckduckgo.app.privacy.db.UserWhitelistDao
 import com.duckduckgo.app.statistics.pixels.Pixel
@@ -48,8 +49,9 @@ class HttpsUpgraderModule {
     fun bloomFilterFactory(
         specificationDao: HttpsBloomFilterSpecDao,
         binaryDataStore: BinaryDataStore,
-        persister: HttpsDataPersister
+        embeddedDataPersister: HttpsEmbeddedDataPersister,
+        dataPersister: HttpsDataPersister
     ): HttpsBloomFilterFactory {
-        return HttpsBloomFilterFactoryImpl(specificationDao, binaryDataStore, persister)
+        return HttpsBloomFilterFactoryImpl(specificationDao, binaryDataStore, embeddedDataPersister, dataPersister)
     }
 }

app/src/main/java/com/duckduckgo/app/httpsupgrade/store/HttpsDataPersister.kt

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020 DuckDuckGo
+ * Copyright (c) 2021 DuckDuckGo
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,15 +16,10 @@
 
 package com.duckduckgo.app.httpsupgrade.store
 
-import android.content.Context
-import com.duckduckgo.app.browser.R
 import com.duckduckgo.app.global.db.AppDatabase
 import com.duckduckgo.app.global.store.BinaryDataStore
 import com.duckduckgo.app.httpsupgrade.model.HttpsBloomFilterSpec
 import com.duckduckgo.app.httpsupgrade.model.HttpsFalsePositiveDomain
-import com.squareup.moshi.JsonAdapter
-import com.squareup.moshi.Moshi
-import com.squareup.moshi.Types
 import timber.log.Timber
 import java.io.IOException
 import javax.inject.Inject
@@ -33,11 +28,16 @@ class HttpsDataPersister @Inject constructor(
     private val binaryDataStore: BinaryDataStore,
     private val httpsBloomSpecDao: HttpsBloomFilterSpecDao,
     private val httpsFalsePositivesDao: HttpsFalsePositivesDao,
-    private val appDatabase: AppDatabase,
-    private val context: Context,
-    private val moshi: Moshi
+    private val appDatabase: AppDatabase
 ) {
 
+    fun persistBloomFilter(specification: HttpsBloomFilterSpec, bytes: ByteArray, falsePositives: List<HttpsFalsePositiveDomain>) {
+        appDatabase.runInTransaction {
+            persistBloomFilter(specification, bytes)
+            persistFalsePositives(falsePositives)
+        }
+    }
+
     fun persistBloomFilter(specification: HttpsBloomFilterSpec, bytes: ByteArray) {
         if (!binaryDataStore.verifyCheckSum(bytes, specification.sha256)) {
             throw IOException("Https binary has incorrect sha, throwing away file")
@@ -50,36 +50,11 @@ class HttpsDataPersister @Inject constructor(
         }
     }
 
-    private fun persistBloomFilter(specification: HttpsBloomFilterSpec, bytes: ByteArray, falsePositives: List<HttpsFalsePositiveDomain>) {
-        appDatabase.runInTransaction {
-            persistBloomFilter(specification, bytes)
-            persistFalsePositives(falsePositives)
-        }
-    }
-
     fun persistFalsePositives(falsePositives: List<HttpsFalsePositiveDomain>) {
         httpsFalsePositivesDao.updateAll(falsePositives)
     }
 
-    fun persistEmbeddedData() {
-        Timber.d("Updating https data from embedded files")
-        val specJson = context.resources.openRawResource(R.raw.https_mobile_v2_bloom_spec).bufferedReader().use { it.readText() }
-        val specAdapter = moshi.adapter(HttpsBloomFilterSpec::class.java)
-
-        val falsePositivesJson = context.resources.openRawResource(R.raw.https_mobile_v2_false_positives).bufferedReader().use { it.readText() }
-        val falsePositivesType = Types.newParameterizedType(List::class.java, HttpsFalsePositiveDomain::class.java)
-        val falsePositivesAdapter: JsonAdapter<List<HttpsFalsePositiveDomain>> = moshi.adapter(falsePositivesType)
-
-        val bytes = context.resources.openRawResource(R.raw.https_mobile_v2_bloom).use { it.readBytes() }
-        persistBloomFilter(specAdapter.fromJson(specJson)!!, bytes, falsePositivesAdapter.fromJson(falsePositivesJson)!!)
-    }
-
     fun isPersisted(specification: HttpsBloomFilterSpec): Boolean {
         return specification == httpsBloomSpecDao.get() && binaryDataStore.verifyCheckSum(HttpsBloomFilterSpec.HTTPS_BINARY_FILE, specification.sha256)
     }
-
-    fun isPersisted(): Boolean {
-        val specification = httpsBloomSpecDao.get() ?: return false
-        return binaryDataStore.verifyCheckSum(HttpsBloomFilterSpec.HTTPS_BINARY_FILE, specification.sha256)
-    }
 }

app/src/main/java/com/duckduckgo/app/httpsupgrade/store/HttpsEmbeddedDataPersister.kt

@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2021 DuckDuckGo
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.duckduckgo.app.httpsupgrade.store
+
+interface HttpsEmbeddedDataPersister {
+
+    fun shouldPersistEmbeddedData(): Boolean
+
+    fun persistEmbeddedData()
+
+}

app/src/play/java/com/duckduckgo/app/di/HttpsPersisterModule.kt

@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2021 DuckDuckGo
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.duckduckgo.app.di
+
+import android.content.Context
+import com.duckduckgo.app.global.store.BinaryDataStore
+import com.duckduckgo.app.httpsupgrade.store.HttpsBloomFilterSpecDao
+import com.duckduckgo.app.httpsupgrade.store.HttpsDataPersister
+import com.duckduckgo.app.httpsupgrade.store.HttpsEmbeddedDataPersister
+import com.duckduckgo.httpsupgrade.store.PlayHttpsEmbeddedDataPersister
+import com.squareup.moshi.Moshi
+import dagger.Module
+import dagger.Provides
+
+@Module
+class HttpsPersisterModule {
+
+    @Provides
+    fun providesPlayHttpsEmbeddedDataPersister(
+        httpsDataPersister: HttpsDataPersister,
+        binaryDataStore: BinaryDataStore,
+        httpsBloomSpecDao: HttpsBloomFilterSpecDao,
+        context: Context,
+        moshi: Moshi
+    ): HttpsEmbeddedDataPersister {
+        return PlayHttpsEmbeddedDataPersister(httpsDataPersister, binaryDataStore, httpsBloomSpecDao, context, moshi)
+    }
+
+}

app/src/play/java/com/duckduckgo/httpsupgrade/store/PlayHttpsEmbeddedDataPersister.kt

@@ -0,0 +1,42 @@
+package com.duckduckgo.httpsupgrade.store
+
+import android.content.Context
+import com.duckduckgo.app.browser.R
+import com.duckduckgo.app.global.store.BinaryDataStore
+import com.duckduckgo.app.httpsupgrade.model.HttpsBloomFilterSpec
+import com.duckduckgo.app.httpsupgrade.model.HttpsFalsePositiveDomain
+import com.duckduckgo.app.httpsupgrade.store.HttpsBloomFilterSpecDao
+import com.duckduckgo.app.httpsupgrade.store.HttpsDataPersister
+import com.duckduckgo.app.httpsupgrade.store.HttpsEmbeddedDataPersister
+import com.squareup.moshi.JsonAdapter
+import com.squareup.moshi.Moshi
+import com.squareup.moshi.Types
+import timber.log.Timber
+
+class PlayHttpsEmbeddedDataPersister(
+    private val httpsDataPersister: HttpsDataPersister,
+    private val binaryDataStore: BinaryDataStore,
+    private val httpsBloomSpecDao: HttpsBloomFilterSpecDao,
+    private val context: Context,
+    private val moshi: Moshi
+) : HttpsEmbeddedDataPersister {
+
+    override fun shouldPersistEmbeddedData(): Boolean {
+        val specification = httpsBloomSpecDao.get() ?: return true
+        return !binaryDataStore.verifyCheckSum(HttpsBloomFilterSpec.HTTPS_BINARY_FILE, specification.sha256)
+    }
+
+    override fun persistEmbeddedData() {
+        Timber.d("Updating https data from embedded files")
+        val specJson = context.resources.openRawResource(R.raw.https_mobile_v2_bloom_spec).bufferedReader().use { it.readText() }
+        val specAdapter = moshi.adapter(HttpsBloomFilterSpec::class.java)
+
+        val falsePositivesJson = context.resources.openRawResource(R.raw.https_mobile_v2_false_positives).bufferedReader().use { it.readText() }
+        val falsePositivesType = Types.newParameterizedType(List::class.java, HttpsFalsePositiveDomain::class.java)
+        val falsePositivesAdapter: JsonAdapter<List<HttpsFalsePositiveDomain>> = moshi.adapter(falsePositivesType)
+
+        val bytes = context.resources.openRawResource(R.raw.https_mobile_v2_bloom).use { it.readBytes() }
+        httpsDataPersister.persistBloomFilter(specAdapter.fromJson(specJson)!!, bytes, falsePositivesAdapter.fromJson(falsePositivesJson)!!)
+    }
+
+}