Schedule metadata import jobs after index imports

.cirrus.yml

@@ -1,8 +1,8 @@
 freebsd_instance:
-  image_family: freebsd-14-2
+  image_family: freebsd-14-3
 
 task:
-  name: "Backend build on FreeBSD 14.2"
+  name: "Backend build on FreeBSD 14.3"
   env:
     GHCUP_VERSION: 0.1.50.2
     CABAL_PROJECT: cabal.project

.github/workflows/backend.yml

@@ -72,9 +72,6 @@ jobs:
         cabal update
         mkdir -p ~/.local/share
         git clone https://github.com/haskell/security-advisories.git ~/.local/share/security-advisories
-        cd ~/.local/share/security-advisories
-        git checkout df64e86a39668c057031fe7e2c679b1003090e03
-        cd -
 
     - name: "Create freeze file"
       run: |

.github/workflows/frontend.yml

@@ -13,7 +13,7 @@ jobs:
 
     - uses: actions/setup-node@v6
       with:
-        node-version: "18"
+        node-version: "20"
         cache: "yarn"
         cache-dependency-path: assets/yarn.lock
 

Dockerfile

@@ -99,7 +99,7 @@ RUN ghcup install ghc $GHC_VERSION
 RUN ghcup set ghc $GHC_VERSION
 
 RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
-RUN curl -fsSL https://deb.nodesource.com/setup_18.x | bash -
+RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash -
 RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
 RUN apt update
 RUN apt install -y direnv \

Makefile

@@ -158,7 +158,7 @@ docker-build: ## Build and start the container cluster
 	@docker compose build devel
 
 docker-up: ## Start the container cluster
-	@docker compose up -d
+	@docker compose up -d --build
 
 docker-stop: ## Stop the container cluster without removing the containers
 	@docker compose stop

cabal.project

@@ -32,17 +32,6 @@ package warp
 package zlib
   flags: -pkg-config
 
-source-repository-package
-  type: git
-  location: https://github.com/haskell/security-advisories/
-  tag: 0452a2180f3c6d3e7875d2f391136ef92c8eab69
-  subdir:
-    ./code/cvss
-    ./code/osv
-    ./code/hsec-core
-    ./code/hsec-sync
-    ./code/hsec-tools
-
 source-repository-package
   type: git
   location: https://github.com/scrive/tracing

cabal.project.freeze

@@ -7,7 +7,7 @@ constraints: any.Cabal ==3.12.1.0,
              JuicyPixels -mmap,
              any.OneTuple ==0.4.2,
              any.Only ==0.1,
-             any.PyF ==0.11.4.0,
+             any.PyF ==0.11.5.0,
              any.QuickCheck ==2.15.0.1,
              QuickCheck -old-random +templatehaskell,
              any.RSA ==2.4.1,
@@ -22,7 +22,7 @@ constraints: any.Cabal ==3.12.1.0,
              any.aeson-pretty ==0.8.10,
              aeson-pretty -lib-only,
              any.alex ==3.5.4.0,
-             any.ansi-terminal ==1.1.3,
+             any.ansi-terminal ==1.1.4,
              ansi-terminal -example,
              any.ansi-terminal-types ==1.1.3,
              any.appar ==0.1.8,
@@ -67,14 +67,14 @@ constraints: any.Cabal ==3.12.1.0,
              any.boring ==0.2.2,
              boring +tagged,
              any.bsb-http-chunked ==0.0.0.4,
-             any.bytebuild ==0.3.16.3,
+             any.bytebuild ==0.3.17.0,
              bytebuild -checked,
              any.byteorder ==1.0.4,
              any.byteslice ==0.2.15.0,
              byteslice +avoid-rawmemchr,
-             any.bytesmith ==0.3.13.0,
+             any.bytesmith ==0.3.14.0,
              any.bytestring ==0.12.2.0,
-             any.cabal-doctest ==1.0.11,
+             any.cabal-doctest ==1.0.12,
              any.call-stack ==0.4.0,
              any.case-insensitive ==1.2.1.0,
              any.cassava ==0.5.4.1,
@@ -121,7 +121,7 @@ constraints: any.Cabal ==3.12.1.0,
              crypton -check_alignment +integer-gmp -old_toolchain_inliner +support_aesni +support_deepseq +support_pclmuldq +support_rdrand -support_sse +use_target_attributes,
              any.crypton-connection ==0.3.2,
              any.crypton-x509 ==1.7.7,
-             any.crypton-x509-store ==1.6.11,
+             any.crypton-x509-store ==1.6.12,
              any.crypton-x509-system ==1.6.7,
              any.crypton-x509-validation ==1.6.14,
              any.cvss ==0.2.0.1,
@@ -144,7 +144,7 @@ constraints: any.Cabal ==3.12.1.0,
              directory-ospath-streaming +os-string,
              any.distributive ==0.6.2.1,
              distributive +semigroups +tagged,
-             any.djot ==0.1.2.3,
+             any.djot ==0.1.2.4,
              any.dlist ==1.0,
              dlist -werror,
              any.doclayout ==0.5.0.1,
@@ -164,7 +164,7 @@ constraints: any.Cabal ==3.12.1.0,
              any.eventlog-socket ==0.1.0.0,
              any.exceptions ==0.10.9,
              any.extensible-exceptions ==0.1.1.4,
-             any.extra ==1.8,
+             any.extra ==1.8.1,
              any.fast-logger ==3.2.6,
              any.feed ==1.3.2.1,
              any.file-embed ==0.0.16.0,
@@ -202,19 +202,18 @@ constraints: any.Cabal ==3.12.1.0,
              any.hdaemonize ==0.5.7,
              any.heaps ==0.4.1,
              any.hedgehog ==1.7,
-             any.heptapod ==1.1.0.0,
+             any.heptapod ==1.1.0.1,
              heptapod -pedantic,
              any.hostname ==1.0,
              any.hourglass ==0.2.12,
              any.hpc ==0.7.0.2,
              any.hsc2hs ==0.68.10,
              hsc2hs -in-ghc-tree,
-             any.hsec-core ==0.2.1.0,
-             any.hsec-sync ==0.2.0.2,
-             any.hsec-tools ==0.3.0.0,
-             any.hspec ==2.11.14,
-             any.hspec-core ==2.11.14,
-             any.hspec-discover ==2.11.14,
+             any.hsec-core ==0.3.0.0,
+             any.hsec-tools ==0.3.0.1,
+             any.hspec ==2.11.16,
+             any.hspec-core ==2.11.16,
+             any.hspec-discover ==2.11.16,
              any.hspec-expectations ==0.8.4,
              any.hsyslog ==5.0.2,
              hsyslog -install-examples,
@@ -229,9 +228,9 @@ constraints: any.Cabal ==3.12.1.0,
              http-conduit +aeson,
              any.http-date ==0.0.11,
              any.http-media ==0.8.1.1,
-             any.http-semantics ==0.3.0,
+             any.http-semantics ==0.4.0,
              any.http-types ==0.12.4,
-             any.http2 ==5.3.10,
+             any.http2 ==5.4.0,
              http2 -devel -h2spec,
              any.indexed-profunctors ==0.1.1.1,
              any.indexed-traversable ==0.1.4,
@@ -250,7 +249,6 @@ constraints: any.Cabal ==3.12.1.0,
              any.kan-extensions ==5.2.7,
              any.lens ==5.3.5,
              lens -benchmark-uniplate -dump-splices +inlining -j +test-hunit +test-properties +test-templates +trustworthy,
-             any.lens-aeson ==1.2.3,
              any.libsodium-bindings ==0.0.3.0,
              libsodium-bindings -homebrew -pkg-config,
              any.libyaml ==0.1.4,
@@ -319,7 +317,6 @@ constraints: any.Cabal ==3.12.1.0,
              parser-combinators -dev,
              any.parsers ==0.12.12,
              parsers +attoparsec +binary +parsec,
-             any.pathwalk ==0.3.1.2,
              any.pcre2 ==2.2.2,
              any.pem ==0.2.4,
              any.pg-entity ==0.0.6.0,
@@ -423,22 +420,22 @@ constraints: any.Cabal ==3.12.1.0,
              splitmix -optimised-mixer,
              any.stm ==2.5.3.1,
              any.stm-chans ==3.0.0.9,
-             any.streaming-commons ==0.2.3.0,
+             any.streaming-commons ==0.2.3.1,
              streaming-commons -use-bytestring-builder,
              any.streamly ==0.11.0,
-             streamly -debug -dev -fusion-plugin -has-llvm -inspection -limit-build-mem +opt -use-unliftio,
+             streamly -debug -fusion-plugin -has-llvm -inspection -internal-dev -internal-use-unliftio -limit-build-mem +opt,
              any.streamly-core ==0.3.0,
-             streamly-core -debug -dev -force-lstat-readdir -has-llvm -limit-build-mem +opt -use-folds -use-unfolds -use-unliftio,
+             streamly-core -debug -force-lstat-readdir -has-llvm -internal-dev -internal-use-unliftio -limit-build-mem +opt -use-folds -use-unfolds,
              any.strict ==0.5.1,
              any.strict-mutable-base ==1.1.0.0,
              any.string-conv ==0.2.0,
              string-conv -lib-werror,
              any.string-conversions ==0.4.0.1,
              any.syb ==0.7.3,
-             any.tagged ==0.8.9,
-             tagged +deepseq +transformers,
+             any.tagged ==0.8.10,
+             tagged +deepseq +template-haskell,
              any.tagsoup ==0.14.8,
-             any.tar ==0.6.4.0,
+             any.tar ==0.7.0.0,
              any.tasty ==1.5.3,
              tasty +unix,
              any.tasty-hunit ==0.10.2,
@@ -456,7 +453,6 @@ constraints: any.Cabal ==3.12.1.0,
              any.text-manipulate ==0.3.1.0,
              any.text-short ==0.1.6,
              text-short -asserts,
-             any.tf-random ==0.5,
              any.th-abstraction ==0.7.1.0,
              any.th-compat ==0.1.6,
              any.th-expand-syns ==0.4.12.0,
@@ -470,7 +466,7 @@ constraints: any.Cabal ==3.12.1.0,
              any.time-compat ==1.9.8,
              any.time-locale-compat ==0.1.1.5,
              time-locale-compat -old-locale,
-             any.time-manager ==0.2.3,
+             any.time-manager ==0.2.4,
              any.timerep ==2.1.0.0,
              any.timing-convenience ==0.1,
              any.tls ==2.0.6,
@@ -492,7 +488,7 @@ constraints: any.Cabal ==3.12.1.0,
              any.typst-symbols ==0.1.8.1,
              any.unicode-collation ==0.1.3.6,
              unicode-collation -doctests -executable,
-             any.unicode-data ==0.6.0,
+             any.unicode-data ==0.8.0,
              unicode-data -dev-has-icu,
              any.unicode-transforms ==0.4.0.1,
              unicode-transforms -bench-show -dev -has-icu -has-llvm -use-gauge,
@@ -504,7 +500,7 @@ constraints: any.Cabal ==3.12.1.0,
              any.unlifted ==0.2.3.0,
              any.unliftio ==0.2.25.1,
              any.unliftio-core ==0.2.1.0,
-             any.unordered-containers ==0.2.20.1,
+             any.unordered-containers ==0.2.21,
              unordered-containers -debug,
              any.uri-bytestring ==0.4.0.1,
              uri-bytestring -lib-werror,
@@ -519,7 +515,7 @@ constraints: any.Cabal ==3.12.1.0,
              any.vector-algorithms ==0.9.1.0,
              vector-algorithms +bench +boundschecks -internalchecks -llvm -unsafechecks,
              any.vector-stream ==0.1.0.1,
-             any.void ==0.7.3,
+             any.void ==0.7.4,
              void -safe,
              any.wai ==3.2.4,
              any.wai-app-static ==3.1.9,
@@ -530,16 +526,14 @@ constraints: any.Cabal ==3.12.1.0,
              any.wai-logger ==2.5.0,
              any.wai-middleware-heartbeat ==0.0.1.0,
              any.wai-middleware-prometheus ==1.0.1.0,
-             any.warp ==3.4.9,
+             any.warp ==3.4.11,
              warp +allow-sendfilefd +include-warp-version -network-bytestring -warp-debug -x509,
              any.wide-word ==0.1.8.1,
              any.witherable ==0.5,
              any.wl-pprint-annotated ==0.1.0.1,
              any.word8 ==0.1.3,
-             any.wreq ==0.5.4.3,
-             wreq -aws -developer +doctest -httpbin,
              any.xml ==1.3.14,
-             any.xml-conduit ==1.10.0.1,
+             any.xml-conduit ==1.10.1.0,
              any.xml-types ==0.3.8,
              any.yaml ==0.11.11.2,
              yaml +no-examples +no-exe,
@@ -548,4 +542,4 @@ constraints: any.Cabal ==3.12.1.0,
              zip-archive -executable,
              any.zlib ==0.7.1.1,
              zlib -bundled-c-zlib +non-blocking-ffi -pkg-config
-index-state: hackage.haskell.org 2025-10-21T14:26:30Z
+index-state: hackage.haskell.org 2025-12-14T18:03:17Z

flora.cabal

@@ -489,6 +489,7 @@ library flora-jobs
   build-depends:
     Cabal-syntax,
     aeson,
+    async,
     base,
     bytestring,
     commonmark,

src/advisories/Advisories/Import.hs

@@ -5,6 +5,7 @@ import Data.Foldable (forM_, traverse_)
 import Data.Function ((&))
 import Data.List.NonEmpty (NonEmpty)
 import Data.List.NonEmpty qualified as NonEmpty
+import Data.Text qualified as Text
 import Data.Text.Display
 import Data.UUID.V4 qualified as UUID
 import Data.Vector (Vector)
@@ -115,7 +116,7 @@ processAffectedPackage advisoryId affected = do
   affectedPackageId <- AffectedPackageId <$> liftIO UUID.nextRandom
   let packageName =
         case affected.affectedComponentIdentifier of
-          Hackage affectedPackageName -> PackageName affectedPackageName
+          Repository _ ((RepositoryName "hackage")) affectedPackageName -> PackageName (Text.pack . unPackageName $ affectedPackageName)
           GHC _ -> PackageName "ghc"
   let namespace = Namespace "hackage"
   package <- guardThatPackageExists namespace packageName $ \_ _ -> do

src/advisories/Advisories/Model/Affected/Types.hs

@@ -14,7 +14,7 @@ import Database.PostgreSQL.Simple.ToField
 import Distribution.Types.Version
 import Distribution.Types.VersionRange (VersionRange)
 import GHC.Generics
-import Security.Advisories.Core.Advisory
+import Security.Advisories.Core.Advisory (Architecture, OS)
 import Security.Advisories.Core.HsecId
 import Security.CVSS (CVSS)
 

src/core/Flora/Import/Package/Bulk/Archive.hs

@@ -15,6 +15,7 @@ import Codec.Compression.GZip qualified as GZip
 import Control.Monad
 import Data.Aeson
 import Data.ByteString (StrictByteString)
+import Data.ByteString.Lazy (LazyByteString)
 import Data.ByteString.Lazy qualified as BL
 import Data.List (isSuffixOf)
 import Data.Maybe (fromMaybe)
@@ -108,7 +109,7 @@ importFromArchive repositoryName indexDependencies indexArchiveBasePath = do
     buildContentStream
       :: UTCTime
       -> Stream (Eff es) (ImportFileType, UTCTime, StrictByteString)
-      -> Tar.GenEntry Tar.TarPath linkTarget
+      -> Tar.GenEntry LazyByteString Tar.TarPath linkTarget
       -> Stream (Eff es) (ImportFileType, UTCTime, StrictByteString)
     buildContentStream time acc entry =
       let entryPath = Tar.entryPath entry

src/jobs-worker/FloraJobs/Runner.hs

@@ -1,5 +1,7 @@
 module FloraJobs.Runner where
 
+import Control.Concurrent (forkIO)
+import Control.Concurrent.Async qualified as Async
 import Control.Exception
 import Control.Monad
 import Control.Monad.IO.Class
@@ -56,7 +58,7 @@ import Flora.Model.Release.Types
 import Flora.Model.Release.Update qualified as Update
 import Flora.Monad
 import FloraJobs.Render (renderMarkdown)
-import FloraJobs.Scheduler (scheduleRefreshIndex)
+import FloraJobs.Scheduler
 import FloraJobs.ThirdParties.Hackage.API
   ( HackagePackageInfo (..)
   , HackagePreferredVersions (..)
@@ -285,9 +287,43 @@ refreshIndex indexName = do
         object ["package_index" .= indexName]
       error $ Text.unpack $ "Package index " <> indexName <> " not found in the database!"
     Just packageIndex -> do
+      pool <- getPool
       indexDependencies <- Query.getIndexDependencies packageIndex.packageIndexId
       Import.importFromArchive indexName indexDependencies packagesPath
-      pool <- getPool
+
+      releasesWithoutReadme <- Query.getHackagePackageReleasesWithoutReadme
+      liftIO $
+        void $
+          forkIO $
+            Async.forConcurrently_
+              releasesWithoutReadme
+              (\(releaseId, version, packagename) -> scheduleReadmeJob pool releaseId packagename version)
+
+      releasesWithoutUploadTime <- Query.getHackagePackageReleasesWithoutUploadTimestamp
+      liftIO $
+        void $
+          forkIO $
+            Async.forConcurrently_
+              releasesWithoutUploadTime
+              (\(releaseId, version, packagename) -> scheduleUploadTimeJob pool releaseId packagename version)
+
+      releasesWithoutChangelog <- Query.getHackagePackageReleasesWithoutChangelog
+      liftIO $
+        void $
+          forkIO $
+            Async.forConcurrently_
+              releasesWithoutChangelog
+              (\(releaseId, version, packagename) -> scheduleChangelogJob pool releaseId packagename version)
+
+      packagesWithoutDeprecationInformation <- Query.getHackagePackagesWithoutReleaseDeprecationInformation
+      liftIO $
+        void $
+          forkIO $ do
+            Async.forConcurrently_
+              packagesWithoutDeprecationInformation
+              (\a -> scheduleReleaseDeprecationListJob pool a)
+            void $ scheduleRefreshLatestVersions pool
+
       void $ liftIO $ scheduleRefreshIndex pool indexName
 
 getCabalPackagesDirectory :: FileSystem :> es => FloraM es FilePath

test/Flora/AdvisorySpec.hs

@@ -45,4 +45,4 @@ testFetchingAdvisoryPreviewByPackageId = do
         (Namespace "hackage")
         (PackageName "base")
   advisories <- Query.getAdvisoryPreviewsByPackageId package.packageId
-  assertEqual_ 1 (Vector.length advisories)
+  assertEqual_ 2 (Vector.length advisories)