Upmerge changes from hoogle.haskell.org

deploy.nix

@@ -0,0 +1,121 @@
+{ hoogle, cores ? 4 }:
+{ lib, pkgs, ... }:
+
+# The Plan:
+# Hoogle serves on a uniquely-named UNIX domain socket which we
+# configure nginx to forward to via the nginxConf configuration
+# fragment. The Hoogle database is periodically updated by
+# rotate-hoogle.service which generates a new Hoogle database,
+# starts a new hoogle server, updates the nginx configuration
+# and shuts down the old server.
+
+let
+  hoogleRun = "/run/hoogle";
+  nginxConf = "${hoogleRun}/nginx.conf";
+  socket = inst: "/run/hoogle/hoogle-${inst}.sock";
+  nServers = 12;
+in
+{
+  users.users.hoogle = {
+    isSystemUser = true;
+    description = "hoogle server";
+    group = "hoogle";
+  };
+
+  users.groups.hoogle = {
+    members = [ "nginx" ];
+  };
+
+  systemd.timers."generate-hoogle" = {
+    description = "Rotate Hoogle instance";
+    timerConfig = {
+      Unit = "generate-hoogle.service";
+      OnCalendar = "hourly";
+      Persistent = true;
+    };
+    wantedBy = [ "timers.target" ];
+  };
+
+  systemd.services = {
+    "generate-hoogle" = {
+      script = ''
+        hoogle generate --database=$DB_DIR/haskell-new.hoo --insecure --download +RTS -N${toString cores} -RTS
+        mv $DB_DIR/haskell-new.hoo $DB_DIR/haskell.hoo
+      '';
+      path = [ hoogle ];
+      after = [ "network.target" ];
+      serviceConfig = {
+        User = "hoogle";
+        Group = "hoogle";
+        Type = "oneshot";
+        TimeoutStartSec = 600;
+        ExecStartPost = "+systemctl restart 'hoogle@*'";
+        BindReadOnlyPaths = [
+          # mount the nix store read-only
+          "/nix/store"
+          # getAppUserDataDirectory needs getUserEntryForID
+          "/etc/passwd"
+        ];
+        PrivateNetwork = false;
+        RuntimeDirectory = "generate-hoogle";
+        StateDirectory = [ "hoogle" ];
+      };
+      environment = {
+        DB_DIR = "%S/hoogle";
+      };
+    };
+
+    "hoogle@" = {
+      script = ''
+        ${hoogle}/bin/hoogle serve \
+          --database=$DB_DIR/haskell.hoo \
+          --scope=set:stackage \
+          --socket=$SOCKET \
+          --links \
+          +RTS -T -RTS;
+      '';
+      serviceConfig = {
+        User = "nginx";
+        Group = "hoogle";
+        BindReadOnlyPaths = [
+          # mount the nix store read-only
+          "/nix/store"
+          # getAppUserDataDirectory needs getUserEntryForID
+          "/etc/passwd"
+          #"/etc/ssl" "/etc/ssl/certs"
+        ];
+        PrivateTmp = false;
+        ProtectSystem = false;
+        ProtectHome = false;
+        NoNewPrivileges = false;
+        Restart = "on-failure";
+        RestartSec = "5s";
+        RuntimeDirectory = "hoogle";
+      };
+      environment = {
+        DB_DIR = "%S/hoogle";
+        SOCKET = socket "%i";
+        SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
+        NIX_SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
+      };
+    };
+  } // (
+    let
+      mkInst = n:
+        lib.attrsets.nameValuePair "hoogle@${toString n}"
+        {
+          wantedBy = [ "multi-user.target" ];
+          overrideStrategy = "asDropin";
+        };
+    in lib.listToAttrs (map mkInst (lib.range 1 nServers))
+  );
+
+  systemd.tmpfiles.rules = [
+    "f ${nginxConf} 0755 nginx nginx -"
+  ];
+
+  services.nginx.upstreams.hoogle.servers =
+    let mkInst = n: lib.attrsets.nameValuePair "unix:${socket (toString n)}" { };
+    in lib.listToAttrs (map mkInst (lib.range 1 nServers));
+}
+

flake.nix

@@ -0,0 +1,55 @@
+{
+  description = "haskell.org hoogle deployment";
+
+  inputs.nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
+  inputs.flake-utils.url = "github:numtide/flake-utils";
+
+  outputs = { self, nixpkgs, flake-utils }:
+    flake-utils.lib.eachDefaultSystem (system:
+      let pkgs = nixpkgs.legacyPackages.${system}; in
+      {
+        packages = rec {
+          hoogle =
+            let
+              hsPkgs = pkgs.haskellPackages.override {
+                overrides = self: super: {
+                  hackage-revdeps = super.hackage-revdeps_0_3;
+                };
+              };
+            in hsPkgs.callCabal2nix "hoogle" ./. { };
+          default = hoogle;
+        };
+        apps = rec {
+          hoogle = flake-utils.lib.mkApp { drv = self.packages.${system}.hoogle; };
+          default = hoogle;
+        };
+      }) // {
+        nixosConfigurations."hoogle-test" = nixpkgs.lib.nixosSystem {
+          system = "x86_64-linux";
+          modules = [
+            { boot.isContainer = true; }
+            (import ./deploy.nix {
+              hoogle = self.packages.x86_64-linux.hoogle;
+              cores = 4;
+            })
+            {
+              services.nginx.virtualHosts."hoogle.haskell.org" = {
+                locations."/" = {
+                  proxyPass = "http://hoogle";
+                };
+                addSSL = true;
+              };
+            }
+          ];
+        };
+
+        nixosModules.hoogle-haskell-org = { pkgs, ... }: {
+          imports = [
+            (import ./deploy.nix {
+              hoogle = self.packages.${pkgs.stdenv.hostPlatform.system}.hoogle;
+              cores = 4;
+            })
+          ];
+        };
+      };
+}

hoogle.cabal

@@ -54,7 +54,7 @@ library
         binary,
         bytestring >= 0.10.2.0,
         Cabal-syntax >= 3.8,
-        cmdargs,
+        optparse-applicative,
         conduit >= 1.3.0,
         conduit-extra >= 1.2.3.2,
         containers >= 0.5,
@@ -74,6 +74,7 @@ library
         js-flot,
         js-jquery,
         mmap,
+        network,
         process-extras,
         resourcet,
         safe >= 0.3.20,
@@ -83,7 +84,9 @@ library
         temporary,
         text >= 2,
         time >= 1.5,
+        tls,
         transformers,
+        streaming-commons,
         uniplate,
         utf8-string >= 0.3.1,
         vector,

misc/Upgrade.hs

@@ -49,9 +49,6 @@ main = do
         echo system_ $ "hoogle_datadir=. " ++ exe ++ " generate --database=haskell.hoo +RTS -M900M -T -N2"
         echo system_ $ "hoogle_datadir=. " ++ exe ++ " test --database=haskell.hoo"
 
-        when False $ -- Frege database has disappeared
-            echo system_ $ "hoogle_datadir=. " ++ exe ++ " generate --database=frege.hoo --frege +RTS -M900M -T -N2"
-
         when False $ do -- DAML now has its own server
             createDirectoryIfMissing True "daml"
             echo system_ "curl https://docs.daml.com/hoogle_db/base.txt --output daml/base.txt"