feat(release): additional q files and infra

Author: sambacha

.editorconfig

@@ -0,0 +1,87 @@
+# top-most EditorConfig file
+root = true
+
+# All files
+[*]
+charset = utf-8
+end_of_line = lf
+indent_size = 2
+indent_style = space
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+# Solidity
+# https://github.com/sambacha/prettier-config-solidity
+[*.sol]
+indent_size = 4
+indent_style = space
+
+# q 
+# kdb+ 
+[*.q]
+indent_style = space
+indent_size = 2
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+# Markdown
+[*.{md,adoc,asciidoc}]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = false
+
+# Match nix files, set indent to spaces with width of two
+[*.nix]
+indent_style = space
+indent_size = 2
+
+# JavaScript, JSON, JSX, JavaScript Modules, TypeScript
+# https://github.com/feross/standard
+# https://prettier.io
+[*.{cjs,js,json,jsx,mjs,ts,tsx,mts,cts}]
+indent_size = 2
+indent_style = space
+
+# TOML
+# https://github.com/toml-lang/toml/tree/master/examples
+[*.toml]
+indent_size = 2
+indent_style = space
+
+# YAML
+# http://yaml.org/spec/1.2/2009-07-21/spec.html#id2576668
+[*.{yaml,yml}]
+indent_size = 2
+indent_style = space
+
+# Shell
+# https://google.github.io/styleguide/shell.xml#Indentation
+[*.{bash,sh,zsh}]
+indent_size = 2
+indent_style = space
+
+# confg + cfg
+[*.{conf,cfg}]
+charset                     = UTF-8
+end_of_line                 = LF
+indent_size                 = 4
+indent_style                = tab
+insert_final_newline        = true
+tab_width                   = 4
+trim_trailing_whitespace    = true
+
+# Match diffs, avoid to trim trailing whitespace
+[*.{diff,patch}]
+trim_trailing_whitespace = false
+
+# Ignore fixtures and vendored files
+[{dist,artifacts,vendor,test/fixtures,tests_config,__snapshot__,}/**]
+charset = unset
+end_of_line = unset
+indent_size = unset
+indent_style = unset
+insert_final_newline = unset
+trim_trailing_spaces = unset

.github/update.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -ex
+
+echo "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" > "hash"
+
+# try (and fail) building, to get SRI hash of cargo deps
+(set +e; nix build |& tee out)
+
+# parse output, get hash, save
+awk '/got: / {print $2}' out > "hash"
+
+function clean() { rm -rf out; }
+trap clean EXIT

.gitignore

@@ -0,0 +1,18 @@
+# -*- mode: gitignore; -*-
+/out
+.DS_Store
+.secret
+.idea/*
+
+logs
+*.log
+.\#*
+**/.terraform/*
+
+# .tfstate files
+*.tfstate
+*.tfstate.*
+*.tfvars
+*.tfvars.json
+*.ssh
+*.key

kfk/0001-Makefile-don-t-download-k.h-from-github-master-branc.patch

@@ -0,0 +1,33 @@
+From 1b9d3667a90358c5d008321818c3e219cc7e30a7 Mon Sep 17 00:00:00 2001
+From: Sam Bacha <[email protected]>
+Date: Sun, 6 Jun 2021 08:25:06 +0200
+Subject: [PATCH] Makefile: don't download k.h from github master branch during
+ build
+
+We provide this after cloning the repo
+---
+ Makefile | 6 ++----
+ 1 file changed, 2 insertions(+), 4 deletions(-)
+
+diff --git a/Makefile b/Makefile
+index 5af64c9..f6a652f 100644
+--- a/Makefile
++++ b/Makefile
+@@ -28,12 +28,10 @@ endif
+ QARCH = $(OSFLAG)$(MS)
+ Q     = $(QHOME)/$(QARCH) 
+ 
+-all: k.h
++all:
+ 	$(CC) kfk.c -m$(MS) $(OPTS) $(LDOPTS_DYNAMIC) $(LD_COMMON) -I$(KFK_INCLUDE) $(LNK) -o $(TGT) $(OSXOPTS)
+-static: k.h
++static:
+ 	$(CC) kfk.c -m$(MS) $(OPTS) $(LDOPTS_STATIC)  $(LD_COMMON) -I$(KFK_INCLUDE) $(LNK) -o $(TGT) $(OSXOPTS)
+-k.h:
+-	curl -s -O -L https://github.com/KxSystems/kdb/raw/master/c/c/k.h 
+ install:
+ 	install $(TGT) $(Q)
+ clean:
+-- 
+2.31.1
+

kfk/default.nix

@@ -0,0 +1,49 @@
+{ stdenv, fetchFromGitHub, rdkafka, zlib, openssl, ... }:
+
+let
+  version = "1.5.0";
+  kdbSrc = fetchFromGitHub {
+    owner = "KxSystems";
+    repo = "kdb";
+    rev = "39b957030bf6a4608f2508ff29894d7fac32a0c2";
+    sha256 = "0r0yfnfn2g89hc7gvk2y1d568lkda0463jw3ha1gg8h83v1vm63f";
+  };
+in
+stdenv.mkDerivation {
+  inherit version;
+  name = "kfk-${version}";
+
+  src = fetchFromGitHub {
+    owner = "KxSystems";
+    repo = "kafka";
+    rev = "v${version}";
+    sha256 = "1yi9f8gy3nv5hncd0qh9q2lzk315pavf3bkc5wq48ki0yxllhaqf";
+  };
+
+  # The Makefile downloaded k.h from kdb repo, master branch.
+  # Patch this out, and copy it into $src after unpack.
+  patches = [ ./0001-Makefile-don-t-download-k.h-from-github-master-branc.patch ];
+
+  preConfigure = ''
+    cp ${kdbSrc}/c/c/k.h .
+  '';
+
+  #NIX_CFLAGS_COMPILE = "-I.";
+  NIX_CFLAGS_LINK = "-L${zlib.out}/lib -L${openssl.out}/lib";
+
+  makeFlags = [
+    "KAFKA_ROOT=${rdkafka}"
+  ];
+
+  # override install phase, cause their `make install` is bonkers, and we only
+  # really care about libkfk.so anyways.
+  installPhase = ''
+    install -Dm644 kfk.q $out/q/kfk.q
+    install -Dm644 libkfk.so $out/q/l64/libkfk.so
+    strip $out/q/l64/*.so
+  '';
+
+  meta = {
+    description = "A thin wrapper for kdb+ around librdkafka C API for Kafka";
+  };
+}

log4q/0001-log4q-use-logfmt-format.patch

@@ -0,0 +1,23 @@
+From fff009102056d071c98f839ed8954d0386bb83eb Mon Sep 17 00:00:00 2001
+From: Sam Bacha <[email protected]>
+Date: Thu, 20 Jan 2022 12:08:01 +0100
+Subject: [PATCH] log4q: use logfmt format
+
+---
+ log4q.q | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/log4q.q b/log4q.q
+index 87f601a..67be65c 100644
+--- a/log4q.q
++++ b/log4q.q
+@@ -1,5 +1,5 @@
+ \d .log4q
+-fm:"%c\t[%p]:H=%h:PID[%i]:%d:%t:%f: %m\r\n";
++fm:"level=%c file=%f host=%h pid=%i %m\n";
+ sev:snk:`SILENT`DEBUG`INFO`WARN`ERROR`FATAL!();a:{$[1<count x;[h[x 0]::x 1;snk[y],::x 0];[h[x]::{x@y};snk[y],::x;]];};r:{snk::@[snk;y;except;x];};
+ h:m:()!();m["c"]:{[x;y]string x};m["f"]:{[x;y]string .z.f};m["p"]:{[x;y]string .z.p};m["P"]:{[x;y]string .z.P};m["m"]:{[x;y]y};m["h"]:{[x;y]string .z.h};m["i"]:{[x;y]string .z.i};m["d"]:{[x;y]string .z.d};m["D"]:{[x;y]string .z.D};m["t"]:{[x;y]string .z.t};m["T"]:{[x;y]string .z.T};
+ l:{ssr/[fm;"%",/:lfm;m[lfm:raze -1_/:2_/:nl where fm like/: nl:"*%",/:(.Q.a,.Q.A),\:"*"].\:(x;y)]};
+-- 
+2.34.1
+

log4q/default.nix

@@ -0,0 +1,18 @@
+{ stdenv, fetchFromGitHub }:
+stdenv.mkDerivation {
+  pname = "log4q";
+  version = "0.0.0+unstable";
+
+  src = fetchFromGitHub {
+    owner = "prodrive11";
+    repo = "log4q";
+    rev = "932f33bfeab3f1faa60aac187d23bacc9bf1bd9e";
+    sha256 = "sha256-Zr/GSicHerr1KGMUBvFlZSkR/HTBgVOp0BjEMYR5seI=";
+  };
+
+  patches = [ ./0001-log4q-use-logfmt-format.patch ];
+
+  installPhase = ''
+    install -D log4q.q $out/q/log4q.q
+  '';
+}

main.nix

@@ -0,0 +1,133 @@
+{ system ? builtins.currentSystem
+  , nixpkgs ? import ./nixpkgs.nix { inherit system; }
+  }:
+  let
+    sources = import ./sources.nix;
+    devshell = import sources.devshell { pkgs = nixpkgs; };
+    tf = nixpkgs.terraform.withPlugins (p: [
+      p.aws
+      p.cloudflare
+      p.external
+      p.grafana
+      (nixpkgs.callPackage ./pkgs/terraform-provider-htpasswd { })
+      p.helm
+      p.http
+      p.kafka
+      p.kubernetes
+      p.local
+      p.null
+      p.random
+      p.secret
+      p.template
+      p.tls
+    ]);
+
+    
+# Q third-party libs
+  kfk = nixpkgs.callPackage ./pkgs/kfk { }; # "kafka-kdb"
+  log4q = nixpkgs.callPackage ./pkgs/log4q { };
+  protobufkdb = nixpkgs.callPackage ./pkgs/protobufkdb { };
+  qAbiEncode = nixpkgs.callPackage ./pkgs/qAbiEncode { };
+  qbigint = nixpkgs.callPackage ./pkgs/qbigint { };
+  qquartic = nixpkgs.callPackage ./pkgs/qquartic { };
+  qunit = nixpkgs.callPackage ./pkgs/qunit { };
+  qwebapi = nixpkgs.callPackage ./pkgs/qwebapi { };
+  # Bundle them all into one big package
+  qVendorLibs = nixpkgs.symlinkJoin {
+    name = "q-libs";
+    paths = [
+      kfk
+      log4q
+      protobufkdb
+      qAbiEncode
+      qbigint
+      qquartic
+      qunit
+      qwebapi
+    ];
+  };
+
+  q = nixpkgs.callPackage ./pkgs/q {
+    # Make those part of the base binary
+    qLibs = qVendorLibs;
+  };
+
+  # kdb source
+  kdbLibs = nixpkgs.symlinkJoin {
+    name = "kdb-libs";
+    paths = [
+      # Get the protobufs
+      (nixpkgs.runCommandNoCC "manifold-protos-q" { } ''
+        mkdir -p $out/q/proto
+        cp ${manifoldProtos}/*.proto $out/q/proto
+      '')
+      # Get the rest of the source
+      (nixpkgs.runCommandNoCC "manifold-q"
+        {
+          src = builtins.path {
+            path = ../modules/kdb/src/main/q;
+            filter = name: type:
+              with nixpkgs.lib;
+              if type == "directory" then
+              # Avoid adding the old DB folder in there
+                ! hasSuffix "/db" name
+              else # regular files
+              # Only add .q and .sh files
+                hasSuffix ".q" name || hasSuffix ".sh" name;
+          };
+        } ''
+        mkdir $out
+        cp -r $src $out/q
+        chmod +w $out/q
+        patchShebangs $out/q
+      '')
+    ];
+  };
+
+    kdb-container = nixpkgs.dockerTools.buildLayeredImage {
+    name = "<% NAME %>";
+    tag = "dev";
+    contents = [
+      nixpkgs.bashInteractive # for debugging
+      nixpkgs.coreutils
+      nixpkgs.gnused
+      nixpkgs.dockerTools.fakeNss # otherwise q just outputs pwuid
+      q
+      kdbLibs
+    ];
+    maxLayers = 15;
+    config = {
+      Cmd = [ "/q/start.sh" ];
+      Env = [
+        "QHOME=/q"
+        "QDB=/qdb"
+      ];
+      Volumes = {
+        "/qdb" = { };
+      };
+      WorkingDir = "/";
+    };
+    extraCommands = ''
+      # make sure /tmp exists
+      mkdir -m 1777 tmp
+    '';
+  };
+
+
+    packages = [
+        # Used to invoke q scripts
+      nixpkgs.rlwrap
+      q
+
+
+    commands = [
+      { name = "gradle"; category = "dev"; help = "gradle build system"; command = ''$PRJ_ROOT/gradlew "$@"''; }
+      { name = "j"; category = "dev"; help = "just a command runner"; command = ''${nixpkgs.just}/bin/just "$@"''; }
+      { category = "dev"; package = nixpkgs.yarn; }
+      { name = "tf"; category = "ops"; help = "terraform alias"; command = ''${tf}/bin/terraform "$@"''; }
+    ];
+  };
+
+
+  nixos-configs = (nixpkgs.lib.recurseIntoAttrs (import ../ops/nixos/configs { inherit (nixpkgs) lib nixos; }));
+}