Add tracker blocker for service workers (#1249)

Author: marcosholgado

app/src/androidTest/java/com/duckduckgo/app/browser/WebViewRequestInterceptorTest.kt

@@ -491,6 +491,50 @@ class WebViewRequestInterceptorTest {
         verify(webView, never()).loadUrl(any())
     }
 
+    @Test
+    fun whenInterceptFromServiceWorkerAndRequestShouldBlockAndNoSurrogateThenCancellingResponseReturned() = runBlocking<Unit> {
+        whenever(mockResourceSurrogates.get(any())).thenReturn(SurrogateResponse(responseAvailable = false))
+
+        configureShouldNotUpgrade()
+        configureShouldBlock()
+        val response = testee.shouldInterceptFromServiceWorker(
+            request = mockRequest,
+            documentUrl = "foo.com"
+        )
+
+        assertCancelledResponse(response)
+    }
+
+    @Test
+    fun whenInterceptFromServiceWorkerAndRequestShouldBlockButThereIsASurrogateThenResponseReturnedContainsTheSurrogateData() = runBlocking<Unit> {
+        val availableSurrogate = SurrogateResponse(
+            scriptId = "testId",
+            responseAvailable = true,
+            mimeType = "application/javascript",
+            jsFunction = "javascript replacement function goes here"
+        )
+        whenever(mockResourceSurrogates.get(any())).thenReturn(availableSurrogate)
+
+        configureShouldNotUpgrade()
+        configureShouldBlock()
+        val response = testee.shouldInterceptFromServiceWorker(
+            request = mockRequest,
+            documentUrl = "foo.com"
+        )
+
+        assertEquals(availableSurrogate.jsFunction.byteInputStream().read(), response!!.data.read())
+    }
+
+    @Test
+    fun whenInterceptFromServiceWorkerAndRequestIsNullThenReturnNull() = runBlocking<Unit> {
+        assertNull(testee.shouldInterceptFromServiceWorker(request = null, documentUrl = "foo.com"))
+    }
+
+    @Test
+    fun whenInterceptFromServiceWorkerAndDocumentUrlIsNullThenReturnNull() = runBlocking<Unit> {
+        assertNull(testee.shouldInterceptFromServiceWorker(request = mockRequest, documentUrl = null))
+    }
+
     private fun assertRequestCanContinueToLoad(response: WebResourceResponse?) {
         assertNull(response)
     }

app/src/androidTest/java/com/duckduckgo/app/browser/serviceworker/BrowserServiceWorkerClientTest.kt

@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2021 DuckDuckGo
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.duckduckgo.app.browser.serviceworker
+
+import android.webkit.WebResourceRequest
+import androidx.test.filters.SdkSuppress
+import com.duckduckgo.app.CoroutineTestRule
+import com.duckduckgo.app.browser.RequestInterceptor
+import com.duckduckgo.app.global.exception.UncaughtExceptionRepository
+import com.duckduckgo.app.runBlocking
+import com.nhaarman.mockitokotlin2.mock
+import com.nhaarman.mockitokotlin2.verify
+import com.nhaarman.mockitokotlin2.whenever
+import kotlinx.coroutines.ExperimentalCoroutinesApi
+import org.junit.Before
+import org.junit.Rule
+import org.junit.Test
+
+@ExperimentalCoroutinesApi
+@SdkSuppress(minSdkVersion = 24)
+class BrowserServiceWorkerClientTest {
+
+    @get:Rule
+    var coroutinesTestRule = CoroutineTestRule()
+
+    private val requestInterceptor: RequestInterceptor = mock()
+    private val uncaughtExceptionRepository: UncaughtExceptionRepository = mock()
+
+    private lateinit var testee: BrowserServiceWorkerClient
+
+    @Before
+    fun setup() {
+        testee = BrowserServiceWorkerClient(requestInterceptor, uncaughtExceptionRepository)
+    }
+
+    @Test
+    fun whenShouldInterceptRequestAndOriginHeaderExistThenSendItToInterceptor() = coroutinesTestRule.runBlocking {
+        val webResourceRequest: WebResourceRequest = mock()
+        whenever(webResourceRequest.requestHeaders).thenReturn(mapOf("Origin" to "example.com"))
+
+        testee.shouldInterceptRequest(webResourceRequest)
+
+        verify(requestInterceptor).shouldInterceptFromServiceWorker(webResourceRequest, "example.com")
+    }
+
+    @Test
+    fun whenShouldInterceptRequestAndOriginHeaderDoesNotExistButRefererExistThenSendItToInterceptor() = coroutinesTestRule.runBlocking {
+        val webResourceRequest: WebResourceRequest = mock()
+        whenever(webResourceRequest.requestHeaders).thenReturn(mapOf("Referer" to "example.com"))
+
+        testee.shouldInterceptRequest(webResourceRequest)
+
+        verify(requestInterceptor).shouldInterceptFromServiceWorker(webResourceRequest, "example.com")
+    }
+
+    @Test
+    fun whenShouldInterceptRequestAndNoOriginOrRefererHeadersExistThenSendNullToInterceptor() = coroutinesTestRule.runBlocking {
+        val webResourceRequest: WebResourceRequest = mock()
+        whenever(webResourceRequest.requestHeaders).thenReturn(mapOf())
+
+        testee.shouldInterceptRequest(webResourceRequest)
+
+        verify(requestInterceptor).shouldInterceptFromServiceWorker(webResourceRequest, null)
+    }
+
+}

app/src/main/java/com/duckduckgo/app/browser/WebViewRequestInterceptor.kt

@@ -44,6 +44,12 @@ interface RequestInterceptor {
         documentUrl: String?,
         webViewClientListener: WebViewClientListener?
     ): WebResourceResponse?
+
+    @WorkerThread
+    suspend fun shouldInterceptFromServiceWorker(
+        request: WebResourceRequest?,
+        documentUrl: String?
+    ): WebResourceResponse?
 }
 
 class WebViewRequestInterceptor(
@@ -112,18 +118,37 @@ class WebViewRequestInterceptor(
             webViewClientListener?.pageHasHttpResources(documentUrl)
         }
 
+        return getWebResourceResponse(request, documentUrl, webViewClientListener)
+    }
+
+    override suspend fun shouldInterceptFromServiceWorker(
+        request: WebResourceRequest?,
+        documentUrl: String?
+    ): WebResourceResponse? {
+
+        if (documentUrl == null) return null
+        if (request == null) return null
+
+        if (TrustedSites.isTrusted(documentUrl)) {
+            return null
+        }
+
+        return getWebResourceResponse(request, documentUrl, null)
+    }
+
+    private fun getWebResourceResponse(request: WebResourceRequest, documentUrl: String?, webViewClientListener: WebViewClientListener?): WebResourceResponse? {
         val trackingEvent = trackingEvent(request, documentUrl, webViewClientListener)
         if (trackingEvent?.blocked == true) {
             trackingEvent.surrogateId?.let { surrogateId ->
                 val surrogate = resourceSurrogates.get(surrogateId)
                 if (surrogate.responseAvailable) {
-                    Timber.d("Surrogate found for $url")
+                    Timber.d("Surrogate found for ${request.url}")
                     webViewClientListener?.surrogateDetected(surrogate)
                     return WebResourceResponse(surrogate.mimeType, "UTF-8", surrogate.jsFunction.byteInputStream())
                 }
             }
 
-            Timber.d("Blocking request $url")
+            Timber.d("Blocking request ${request.url}")
             privacyProtectionCountDao.incrementBlockedTrackerCount()
             return WebResourceResponse(null, null, null)
         }

app/src/main/java/com/duckduckgo/app/browser/di/BrowserModule.kt

@@ -36,6 +36,7 @@ import com.duckduckgo.app.browser.favicon.FaviconPersister
 import com.duckduckgo.app.browser.favicon.FileBasedFaviconPersister
 import com.duckduckgo.app.browser.httpauth.WebViewHttpAuthStore
 import com.duckduckgo.app.browser.logindetection.*
+import com.duckduckgo.app.browser.serviceworker.ServiceWorkerLifecycleObserver
 import com.duckduckgo.app.browser.session.WebViewSessionInMemoryStorage
 import com.duckduckgo.app.browser.session.WebViewSessionStorage
 import com.duckduckgo.app.browser.tabpreview.FileBasedWebViewPreviewGenerator
@@ -312,4 +313,9 @@ class BrowserModule {
     fun thirdPartyCookieManager(cookieManager: CookieManager, authCookiesAllowedDomainsRepository: AuthCookiesAllowedDomainsRepository): ThirdPartyCookieManager {
         return AppThirdPartyCookieManager(cookieManager, authCookiesAllowedDomainsRepository)
     }
+
+    @Provides
+    @Singleton
+    @IntoSet
+    fun serviceWorkerLifecycleObserver(serviceWorkerLifecycleObserver: ServiceWorkerLifecycleObserver): LifecycleObserver = serviceWorkerLifecycleObserver
 }

app/src/main/java/com/duckduckgo/app/browser/serviceworker/BrowserServiceWorkerClient.kt

@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2021 DuckDuckGo
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.duckduckgo.app.browser.serviceworker
+
+import android.os.Build
+import android.webkit.ServiceWorkerClient
+import android.webkit.WebResourceRequest
+import android.webkit.WebResourceResponse
+import androidx.annotation.RequiresApi
+import com.duckduckgo.app.browser.RequestInterceptor
+import com.duckduckgo.app.global.exception.UncaughtExceptionRepository
+import com.duckduckgo.app.global.exception.UncaughtExceptionSource
+import kotlinx.coroutines.runBlocking
+import timber.log.Timber
+
+@RequiresApi(Build.VERSION_CODES.N)
+class BrowserServiceWorkerClient(private val requestInterceptor: RequestInterceptor, private val uncaughtExceptionRepository: UncaughtExceptionRepository) : ServiceWorkerClient() {
+
+    override fun shouldInterceptRequest(request: WebResourceRequest): WebResourceResponse? {
+        return runBlocking {
+            try {
+                val documentUrl: String? = request.requestHeaders[HEADER_ORIGIN] ?: request.requestHeaders[HEADER_REFERER]
+                Timber.v("Intercepting Service Worker resource ${request.url} type:${request.method} on page $documentUrl")
+                requestInterceptor.shouldInterceptFromServiceWorker(request, documentUrl)
+            } catch (e: Throwable) {
+                uncaughtExceptionRepository.recordUncaughtException(e, UncaughtExceptionSource.SHOULD_INTERCEPT_REQUEST_FROM_SERVICE_WORKER)
+                throw e
+            }
+        }
+    }
+
+    companion object {
+        const val HEADER_ORIGIN = "Origin"
+        const val HEADER_REFERER = "Referer"
+    }
+}

app/src/main/java/com/duckduckgo/app/browser/serviceworker/ServiceWorkerLifecycleObserver.kt

@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2021 DuckDuckGo
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.duckduckgo.app.browser.serviceworker
+
+import android.os.Build
+import android.webkit.ServiceWorkerController
+import androidx.lifecycle.Lifecycle
+import androidx.lifecycle.LifecycleObserver
+import androidx.lifecycle.OnLifecycleEvent
+import com.duckduckgo.app.browser.RequestInterceptor
+import com.duckduckgo.app.global.exception.UncaughtExceptionRepository
+import javax.inject.Inject
+import javax.inject.Singleton
+
+@Singleton
+class ServiceWorkerLifecycleObserver @Inject constructor(
+    private val requestInterceptor: RequestInterceptor,
+    private val uncaughtExceptionRepository: UncaughtExceptionRepository
+) : LifecycleObserver {
+
+    @OnLifecycleEvent(Lifecycle.Event.ON_START)
+    fun setServiceWorkerClient() {
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
+            ServiceWorkerController.getInstance().setServiceWorkerClient(
+                BrowserServiceWorkerClient(requestInterceptor, uncaughtExceptionRepository)
+            )
+        }
+    }
+}

common/src/main/java/com/duckduckgo/app/global/exception/UncaughtExceptionDao.kt

@@ -40,6 +40,7 @@ abstract class UncaughtExceptionDao {
 enum class UncaughtExceptionSource {
     GLOBAL,
     SHOULD_INTERCEPT_REQUEST,
+    SHOULD_INTERCEPT_REQUEST_FROM_SERVICE_WORKER,
     ON_PAGE_STARTED,
     ON_PAGE_FINISHED,
     SHOULD_OVERRIDE_REQUEST,

statistics/src/main/java/com/duckduckgo/app/statistics/api/OfflinePixelSender.kt

@@ -118,6 +118,7 @@ class OfflinePixelSender @Inject constructor(
         return when (exception.exceptionSource) {
             GLOBAL -> APPLICATION_CRASH_GLOBAL
             SHOULD_INTERCEPT_REQUEST -> APPLICATION_CRASH_WEBVIEW_SHOULD_INTERCEPT
+            SHOULD_INTERCEPT_REQUEST_FROM_SERVICE_WORKER -> APPLICATION_CRASH_WEBVIEW_SHOULD_INTERCEPT_SERVICE_WORKER
             ON_PAGE_STARTED -> APPLICATION_CRASH_WEBVIEW_PAGE_STARTED
             ON_PAGE_FINISHED -> APPLICATION_CRASH_WEBVIEW_PAGE_FINISHED
             SHOULD_OVERRIDE_REQUEST -> APPLICATION_CRASH_WEBVIEW_OVERRIDE_REQUEST

statistics/src/main/java/com/duckduckgo/app/statistics/pixels/Pixel.kt

@@ -40,6 +40,7 @@ interface Pixel {
 
         APPLICATION_CRASH("m_d_ac"),
         APPLICATION_CRASH_GLOBAL("m_d_ac_g"),
+        APPLICATION_CRASH_WEBVIEW_SHOULD_INTERCEPT_SERVICE_WORKER("m_d_ac_wisw"),
         APPLICATION_CRASH_WEBVIEW_SHOULD_INTERCEPT("m_d_ac_wi"),
         APPLICATION_CRASH_WEBVIEW_PAGE_STARTED("m_d_ac_wps"),
         APPLICATION_CRASH_WEBVIEW_PAGE_FINISHED("m_d_ac_wpf"),