feat(virustotal): integrate VirusTotal checks into download and install processes

Author: felicijus

lib/download.ps1

@@ -1,8 +1,11 @@
 # Description: Functions for downloading files
 
+. "$PSScriptRoot\..\lib\core.ps1"
+. "$PSScriptRoot\..\lib\virustotal.ps1"
+
 ## Meta downloader
 
-function Invoke-ScoopDownload ($app, $version, $manifest, $bucket, $architecture, $dir, $use_cache = $true, $check_hash = $true) {
+function Invoke-ScoopDownload ($app, $version, $manifest, $bucket, $architecture, $dir, $use_cache = $true, $check_hash = $true, $check_virustotal = $false) {
     # we only want to show this warning once
     if (!$use_cache) { warn 'Cache is being ignored.' }
 
@@ -12,11 +15,40 @@ function Invoke-ScoopDownload ($app, $version, $manifest, $bucket, $architecture
     # can be multiple cookies: they will be used for all HTTP requests.
     $cookies = $manifest.cookie
 
+    # Pre-check all URLs with VirusTotal before downloading
+    $api_key = Get-VirusTotalApiKey
+
+    $safe_urls = @()
+    if ($check_virustotal) {
+        foreach ($url in $urls) {
+            $hash = hash_for_url $manifest $url $architecture
+            $reports = Check-VirusTotalUrl $app $url $hash $api_key $false
+            $reports | ForEach-Object {
+                $file_report = $_
+                $url = $file_report.'App.Url'
+
+                $maliciousResults = $file_report.'FileReport.Malicious'
+                $suspiciousResults = $file_report.'FileReport.Suspicious'
+                if ($maliciousResults -gt 0 -or $suspiciousResults -gt 0) {
+                    warn "$app`: One or more VirusTotal checks failed. Aborting before download."
+                } else {
+                    info "$app`: Safe URL: $url"
+                    $safe_urls += $url
+                }
+            }
+        }
+        if ($safe_urls.Count -eq 0) {
+            abort "No URL passed VirusTotal check for $app. Aborting before download."
+        }
+    } else {
+        $safe_urls = $urls
+    }
+
     # download first
     if (Test-Aria2Enabled) {
         Invoke-CachedAria2Download $app $version $manifest $architecture $dir $cookies $use_cache $check_hash
     } else {
-        foreach ($url in $urls) {
+        foreach ($url in $safe_urls) {
             $fname = url_filename $url
 
             try {
@@ -45,7 +77,7 @@ function Invoke-ScoopDownload ($app, $version, $manifest, $bucket, $architecture
         }
     }
 
-    return $urls.ForEach({ url_filename $_ })
+    return $safe_urls.ForEach({ url_filename $_ })
 }
 
 ## [System.Net] downloader
@@ -457,9 +489,9 @@ function Invoke-CachedAria2Download ($app, $version, $manifest, $architecture, $
             warn "Download failed! (Error $lastexitcode) $(aria_exit_code $lastexitcode)"
             warn $urlstxt_content
             warn $aria2
-            warn $(new_issue_msg $app $bucket "download via aria2 failed")
+            warn $(new_issue_msg $app $bucket 'download via aria2 failed')
 
-            Write-Host "Fallback to default downloader ..."
+            Write-Host 'Fallback to default downloader ...'
 
             try {
                 foreach ($url in $urls) {
@@ -666,7 +698,7 @@ function get_magic_bytes_pretty($file, $glue = ' ') {
     return (get_magic_bytes $file | ForEach-Object { $_.ToString('x2') }) -join $glue
 }
 
-Function Get-RemoteFileSize ($Uri) {
+function Get-RemoteFileSize ($Uri) {
     $response = Invoke-WebRequest -Uri $Uri -Method HEAD -UseBasicParsing
     if (!$response.Headers.StatusCode) {
         $response.Headers.'Content-Length' | ForEach-Object { [int]$_ }
@@ -689,13 +721,13 @@ function url_remote_filename($url) {
     # this function extracts the original filename from the URL.
     $uri = (New-Object URI $url)
     $basename = Split-Path $uri.PathAndQuery -Leaf
-    If ($basename -match '.*[?=]+([\w._-]+)') {
+    if ($basename -match '.*[?=]+([\w._-]+)') {
         $basename = $matches[1]
     }
-    If (($basename -notlike '*.*') -or ($basename -match '^[v.\d]+$')) {
+    if (($basename -notlike '*.*') -or ($basename -match '^[v.\d]+$')) {
         $basename = Split-Path $uri.AbsolutePath -Leaf
     }
-    If (($basename -notlike '*.*') -and ($uri.Fragment -ne '')) {
+    if (($basename -notlike '*.*') -and ($uri.Fragment -ne '')) {
         $basename = $uri.Fragment.Trim('/', '#')
     }
     return $basename

lib/install.ps1

@@ -5,7 +5,7 @@ function nightly_version($quiet = $false) {
     return "nightly-$(Get-Date -Format 'yyyyMMdd')"
 }
 
-function install_app($app, $architecture, $global, $suggested, $use_cache = $true, $check_hash = $true) {
+function install_app($app, $architecture, $global, $suggested, $use_cache = $true, $check_hash = $true,  $check_virustotal = $false) {
     $app, $manifest, $bucket, $url = Get-Manifest $app
 
     if (!$manifest) {
@@ -49,7 +49,7 @@ function install_app($app, $architecture, $global, $suggested, $use_cache = $tru
     $original_dir = $dir # keep reference to real (not linked) directory
     $persist_dir = persistdir $app $global
 
-    $fname = Invoke-ScoopDownload $app $version $manifest $bucket $architecture $dir $use_cache $check_hash
+    $fname = Invoke-ScoopDownload $app $version $manifest $bucket $architecture $dir $use_cache $check_hash $check_virustotal
     Invoke-Extraction -Path $dir -Name $fname -Manifest $manifest -ProcessorArchitecture $architecture
     Invoke-HookScript -HookType 'pre_install' -Manifest $manifest -ProcessorArchitecture $architecture
 

lib/virustotal.ps1

@@ -1,5 +1,5 @@
 . "$PSScriptRoot\json.ps1" # 'json_path'
-. "$PSScriptRoot\download.ps1" # 'hash_for_url'
+# . "$PSScriptRoot\download.ps1" # 'hash_for_url'
 
 # Error codes
 $script:_ERR_UNSAFE = 2
@@ -314,3 +314,16 @@ function virustotal_check_app($app, $manifest, $architecture, $api_key, $scan) {
         Check-VirusTotalUrl $app $url $hash $api_key $scan
     }
 }
+
+function hash_for_url($manifest, $url, $arch) {
+    $hashes = @(hash $manifest $arch) | Where-Object { $_ -ne $null }
+
+    if ($hashes.length -eq 0) { return $null }
+
+    $urls = @(script:url $manifest $arch)
+
+    $index = [array]::IndexOf($urls, $url)
+    if ($index -eq -1) { abort "Couldn't find hash in manifest for '$url'." }
+
+    @($hashes)[$index]
+}

libexec/scoop-config.ps1

@@ -92,6 +92,9 @@
 #       API key used for uploading/scanning files using virustotal.
 #       See: 'https://support.virustotal.com/hc/en-us/articles/115002088769-Please-give-me-an-API-key'
 #
+# use_virustotal: $true|$false
+#       When set to $true, Scoop will always use VirusTotal to scan files after downloading.
+#
 # cat_style:
 #       When set to a non-empty string, Scoop will use 'bat' to display the manifest for
 #       the `scoop cat` command and while doing manifest review. This requires 'bat' to be

libexec/scoop-install.ps1

@@ -24,6 +24,7 @@
 #   -i, --independent               Don't install dependencies automatically
 #   -k, --no-cache                  Don't use the download cache
 #   -s, --skip-hash-check           Skip hash validation (use with caution!)
+#   -w, --virustotal-check          Check the download against VirusTotal (may be slow)
 #   -u, --no-update-scoop           Don't update Scoop before installing if it's outdated
 #   -a, --arch <32bit|64bit|arm64>  Use the specified architecture, if the app supports it
 
@@ -43,11 +44,12 @@ if (get_config USE_SQLITE_CACHE) {
     . "$PSScriptRoot\..\lib\database.ps1"
 }
 
-$opt, $apps, $err = getopt $args 'giksua:' 'global', 'independent', 'no-cache', 'skip-hash-check', 'no-update-scoop', 'arch='
+$opt, $apps, $err = getopt $args 'gikswua:' 'global', 'independent', 'no-cache', 'skip-hash-check', 'virustotal-check', 'no-update-scoop', 'arch='
 if ($err) { error "scoop install: $err"; exit 1 }
 
 $global = $opt.g -or $opt.global
 $check_hash = !($opt.s -or $opt.'skip-hash-check')
+$check_virustotal = $opt.w -or $opt.'virustotal-check' -or (get_config USE_VIRUSTOTAL $false)
 $independent = $opt.i -or $opt.independent
 $use_cache = !($opt.k -or $opt.'no-cache')
 $architecture = Get-DefaultArchitecture
@@ -132,7 +134,7 @@ if ((Test-Aria2Enabled) -and (get_config 'aria2-warning-enabled' $true)) {
     warn "Should it cause issues, run 'scoop config aria2-enabled false' to disable it."
     warn "To disable this warning, run 'scoop config aria2-warning-enabled false'."
 }
-$apps | ForEach-Object { install_app $_ $architecture $global $suggested $use_cache $check_hash }
+$apps | ForEach-Object { install_app $_ $architecture $global $suggested $use_cache $check_hash $check_virustotal }
 
 show_suggestions $suggested
 

libexec/scoop-update.ps1

@@ -11,6 +11,7 @@
 #   -i, --independent      Don't install dependencies automatically
 #   -k, --no-cache         Don't use the download cache
 #   -s, --skip-hash-check  Skip hash validation (use with caution!)
+#   -w, --virustotal-check Check the download against VirusTotal (may be slow)
 #   -q, --quiet            Hide extraneous messages
 #   -a, --all              Update all apps (alternative to '*')
 
@@ -29,11 +30,12 @@ if (get_config USE_SQLITE_CACHE) {
     . "$PSScriptRoot\..\lib\database.ps1"
 }
 
-$opt, $apps, $err = getopt $args 'gfiksqa' 'global', 'force', 'independent', 'no-cache', 'skip-hash-check', 'quiet', 'all'
+$opt, $apps, $err = getopt $args 'gfikswqa' 'global', 'force', 'independent', 'no-cache', 'skip-hash-check', 'virustotal-check', 'quiet', 'all'
 if ($err) { error "scoop update: $err"; exit 1 }
 $global = $opt.g -or $opt.global
 $force = $opt.f -or $opt.force
 $check_hash = !($opt.s -or $opt.'skip-hash-check')
+$check_virustotal = $opt.w -or $opt.'virustotal-check' -or (get_config USE_VIRUSTOTAL $false)
 $use_cache = !($opt.k -or $opt.'no-cache')
 $quiet = $opt.q -or $opt.quiet
 $independent = $opt.i -or $opt.independent
@@ -258,7 +260,7 @@ function Sync-Bucket {
     }
 }
 
-function update($app, $global, $quiet = $false, $independent, $suggested, $use_cache = $true, $check_hash = $true) {
+function update($app, $global, $quiet = $false, $independent, $suggested, $use_cache = $true, $check_hash = $true, $check_virustotal = $false) {
     $old_version = Select-CurrentVersion -AppName $app -Global:$global
     $old_manifest = installed_manifest $app $old_version $global
     $install = install_info $app $old_version $global
@@ -300,6 +302,38 @@ function update($app, $global, $quiet = $false, $independent, $suggested, $use_c
     }
     #endregion Workaround for #2952
 
+    # can be multiple urls: if there are, then installer should go first to make 'installer.args' section work
+    $urls = @(script:url $manifest $architecture)
+
+    # Pre-check all URLs with VirusTotal before downloading
+    $api_key = Get-VirusTotalApiKey
+
+    $safe_urls = @()
+    if ($check_virustotal) {
+        foreach ($url in $urls) {
+            $hash = hash_for_url $manifest $url $architecture
+            $reports = Check-VirusTotalUrl $app $url $hash $api_key $false
+            $reports | ForEach-Object {
+                $file_report = $_
+                $url = $file_report.'App.Url'
+
+                $maliciousResults = $file_report.'FileReport.Malicious'
+                $suspiciousResults = $file_report.'FileReport.Suspicious'
+                if ($maliciousResults -gt 0 -or $suspiciousResults -gt 0) {
+                    warn "$app`: One or more VirusTotal checks failed. Aborting before download."
+                } else {
+                    info "$app`: Safe URL: $url"
+                    $safe_urls += $url
+                }
+            }
+        }
+        if ($safe_urls.Count -eq 0) {
+            abort "No URL passed VirusTotal check for $app. Aborting before download."
+        }
+    } else {
+        $safe_urls = $urls
+    }
+
     # region Workaround
     # Workaround for https://github.com/ScoopInstaller/Scoop/issues/2220 until install is refactored
     # Remove and replace whole region after proper fix
@@ -309,7 +343,7 @@ function update($app, $global, $quiet = $false, $independent, $suggested, $use_c
     } else {
         $urls = script:url $manifest $architecture
 
-        foreach ($url in $urls) {
+        foreach ($url in $safe_urls) {
             Invoke-CachedDownload $app $version $url $null $manifest.cookie $true
 
             if ($check_hash) {
@@ -376,12 +410,12 @@ function update($app, $global, $quiet = $false, $independent, $suggested, $use_c
     }
 
     if ($independent) {
-        install_app $app $architecture $global $suggested $use_cache $check_hash
+        install_app $app $architecture $global $suggested $use_cache $check_hash $check_virustotal
     } else {
         # Also add missing dependencies
         $apps = @(Get-Dependency $app $architecture) -ne $app
         ensure_none_failed $apps
-        $apps.Where({ !(installed $_) }) + $app | ForEach-Object { install_app $_ $architecture $global $suggested $use_cache $check_hash }
+        $apps.Where({ !(installed $_) }) + $app | ForEach-Object { install_app $_ $architecture $global $suggested $use_cache $check_hash $check_virustotal}
     }
 }
 
@@ -462,7 +496,7 @@ if (-not ($apps -or $all)) {
 
     $suggested = @{}
     # $outdated is a list of ($app, $global) tuples
-    $outdated | ForEach-Object { update @_ $quiet $independent $suggested $use_cache $check_hash }
+    $outdated | ForEach-Object { update @_ $quiet $independent $suggested $use_cache $check_hash $check_virustotal }
 }
 
 exit 0