(feature): wip: mongo engine and refactoring

Author: Andreas Drobisch

docker-compose.yaml

@@ -0,0 +1,14 @@
+version: "3.8"
+
+services:
+  vault-server:
+    network_mode: "host"
+    image: vault:latest
+    environment:
+      VAULT_ADDR: "http://0.0.0.0:8200"
+      VAULT_DEV_ROOT_TOKEN_ID: "vault-plaintext-root-token"
+    cap_add:
+      - IPC_LOCK
+  mongo:
+    network_mode: "host"
+    image: mongo

src/main/scala/com/drobisch/tresor/vault/AWS.scala

@@ -19,8 +19,8 @@ final case class AwsContext(
   * @tparam F
   *   effect type to use
   */
-class AWS[F[_]](implicit sync: Sync[F], clock: Clock[F])
-    extends SecretEngineProvider[F, (AwsContext, VaultConfig)] {
+class AWS[F[_]](val path: String)(implicit sync: Sync[F], clock: Clock[F])
+    extends SecretEngineProvider[F, (AwsContext, VaultConfig), Nothing] {
 
   /** create a aws engine credential
     *
@@ -37,10 +37,10 @@ class AWS[F[_]](implicit sync: Sync[F], clock: Clock[F])
     sync.flatMap(sync.delay {
       val roleArnPart = awsContext.roleArn.map(s"&role_arn=" + _).getOrElse("")
       val ttlPart = "&ttl=" + awsContext.ttlString.getOrElse("3600s")
-      val infixPart = if (awsContext.useSts) "sts" else "creds"
+      val stsOrCreds = if (awsContext.useSts) "sts" else "creds"
 
       val requestUri =
-        s"${vaultConfig.apiUrl}/aws/$infixPart/${awsContext.name}?$roleArnPart$ttlPart"
+        s"${vaultConfig.apiUrl}/$path/$stsOrCreds/${awsContext.name}?$roleArnPart$ttlPart"
 
       basicRequest
         .get(uri"$requestUri")
@@ -60,5 +60,5 @@ class AWS[F[_]](implicit sync: Sync[F], clock: Clock[F])
 }
 
 object AWS {
-  def apply[F[_]](implicit sync: Sync[F], clock: Clock[F]) = new AWS[F]
+  def apply[F[_]](path: String)(implicit sync: Sync[F], clock: Clock[F]) = new AWS[F](path)
 }

src/main/scala/com/drobisch/tresor/vault/Database.scala

@@ -2,6 +2,7 @@ package com.drobisch.tresor.vault
 
 import cats.effect.{Clock, Sync}
 import com.drobisch.tresor.Secret
+import io.circe.Json
 import sttp.client3._
 
 final case class DatabaseContext(role: String)
@@ -13,8 +14,8 @@ final case class DatabaseContext(role: String)
   * @tparam F
   *   context type to use
   */
-class Database[F[_]](implicit sync: Sync[F], clock: Clock[F])
-    extends SecretEngineProvider[F, (DatabaseContext, VaultConfig)] {
+class Database[F[_]](val path: String)(implicit sync: Sync[F], clock: Clock[F])
+    extends SecretEngineProvider[F, (DatabaseContext, VaultConfig), Json] {
 
   /** read the credentials for a DB role
     *
@@ -29,7 +30,7 @@ class Database[F[_]](implicit sync: Sync[F], clock: Clock[F])
     val (db, vaultConfig) = context
 
     val response = basicRequest
-      .get(uri"${vaultConfig.apiUrl}/database/creds/${db.role}")
+      .get(uri"${vaultConfig.apiUrl}/$path/creds/${db.role}")
       .header("X-Vault-Token", vaultConfig.token)
       .send(backend)
 
@@ -40,5 +41,5 @@ class Database[F[_]](implicit sync: Sync[F], clock: Clock[F])
 }
 
 object Database {
-  def apply[F[_]](implicit sync: Sync[F], clock: Clock[F]) = new Database[F]
+  def apply[F[_]](path: String)(implicit sync: Sync[F], clock: Clock[F]) = new Database[F](path)
 }

src/main/scala/com/drobisch/tresor/vault/HttpSupport.scala

@@ -1,7 +1,16 @@
 package com.drobisch.tresor.vault
 
+import cats.effect.Sync
+import io.circe.Json
 import sttp.client3._
 
 trait HttpSupport {
   protected lazy val backend = HttpURLConnectionBackend()
+
+  implicit class ResponseOps[F[_]](response: Response[Either[String, String]])(implicit sync: Sync[F]) {
+    def parseJson: F[Json] = sync.fromEither(
+      response.body.left
+        .map(httpError => new RuntimeException(s"error during http request: $httpError ($response)"))
+        .flatMap(bodyString => io.circe.parser.decode[Json](bodyString).left.map(parseError => new RuntimeException(s"unable to parse json from $bodyString: $parseError"))))
+  }
 }

src/main/scala/com/drobisch/tresor/vault/KV.scala

@@ -13,8 +13,8 @@ final case class KeyValueContext(key: String)
   * @tparam F
   *   effect type to use
   */
-class KV[F[_]](implicit sync: Sync[F], clock: Clock[F])
-    extends SecretEngineProvider[F, (KeyValueContext, VaultConfig)] {
+class KV[F[_]](val path: String)(implicit sync: Sync[F], clock: Clock[F])
+    extends SecretEngineProvider[F, (KeyValueContext, VaultConfig), Nothing] {
 
   /** read the secret from a path
     *
@@ -29,7 +29,7 @@ class KV[F[_]](implicit sync: Sync[F], clock: Clock[F])
     val (kv, vaultConfig) = context
 
     val response = basicRequest
-      .get(uri"${vaultConfig.apiUrl}/secret/${kv.key}")
+      .get(uri"${vaultConfig.apiUrl}/$path/${kv.key}")
       .header("X-Vault-Token", vaultConfig.token)
       .send(backend)
 
@@ -40,5 +40,5 @@ class KV[F[_]](implicit sync: Sync[F], clock: Clock[F])
 }
 
 object KV {
-  def apply[F[_]](implicit sync: Sync[F], clock: Clock[F]) = new KV[F]
+  def apply[F[_]](path: String)(implicit sync: Sync[F], clock: Clock[F]) = new KV[F](path)
 }

src/main/scala/com/drobisch/tresor/vault/SecretEngineProvider.scala

@@ -1,18 +1,19 @@
 package com.drobisch.tresor.vault
 
-import java.util.concurrent.TimeUnit
-
 import cats.data.ReaderT
+import cats.effect.concurrent.Ref
+import cats.effect.{Clock, IO, Sync}
+import cats.syntax.apply._
 import cats.syntax.flatMap._
 import cats.syntax.functor._
-import cats.syntax.apply._
-import cats.effect.{Clock, Sync}
-import cats.effect.concurrent.Ref
 import com.drobisch.tresor.Provider
-import sttp.client3._
-import io.circe.Json
+import io.circe.{Encoder, Json}
 import io.circe.generic.auto._
+import io.circe.syntax._
 import org.slf4j.LoggerFactory
+import sttp.client3._
+
+import java.util.concurrent.TimeUnit
 
 private[vault] final case class LeaseDTO(
     lease_id: Option[String],
@@ -21,13 +22,37 @@ private[vault] final case class LeaseDTO(
     data: Option[Map[String, Option[String]]]
 )
 
-abstract class SecretEngineProvider[Effect[_], ProviderContext](implicit
+abstract class SecretEngineProvider[Effect[_], ProviderContext, Config](implicit
     sync: Sync[Effect],
     clock: Clock[Effect]
 ) extends Provider[Effect, ProviderContext, Lease]
     with HttpSupport {
   protected val log = LoggerFactory.getLogger(getClass)
 
+  /**
+    * the path at which this engine is mounted
+    * @return
+    */
+  def path: String
+
+  /**
+    * write a config for this engine path
+    * @param config
+    */
+  def writeConfig(name: String, config: Config)(implicit encoder: Encoder[Config]): ReaderT[Effect, VaultConfig, Json] = for {
+    response <-
+      ReaderT[Effect, VaultConfig, Response[Either[String, String]]] { vaultConfig =>
+        val request = basicRequest
+          .post(uri"${vaultConfig.apiUrl}/$path/config/$name")
+          .body(config.asJson.noSpaces)
+          .header("X-Vault-Token", vaultConfig.token)
+          .send(backend)
+
+        sync.delay(request)
+      }
+    json <- ReaderT.liftF(response.parseJson)
+  } yield json
+
   /** renew a lease
     *
     * https://www.vaultproject.io/api/system/leases.html#renew-lease
@@ -36,6 +61,7 @@ abstract class SecretEngineProvider[Effect[_], ProviderContext](implicit
     *   a lease
     * @param increment
     *   time to extend the lease in seconds
+    *   (depending on the engine, this might be treated as the new ttl of the lease, this is the case for Mongo Atlas)
     * @return
     *   reader for an extended lease
     */
@@ -44,40 +70,52 @@ abstract class SecretEngineProvider[Effect[_], ProviderContext](implicit
       increment: Option[Long]
   ): ReaderT[Effect, VaultConfig, Lease] = lease.leaseId
     .map { leaseId =>
-      ReaderT[Effect, VaultConfig, Lease] { vaultConfig =>
-        sync.flatMap(sync.delay {
-          basicRequest
-            .post(uri"${vaultConfig.apiUrl}/sys/leases/renew")
-            .body(
-              Json
-                .obj(
-                  "lease_id" -> Json.fromString(leaseId),
-                  "increment" -> Json.fromLong(increment.getOrElse(3600))
-                )
-                .noSpaces
-            )
-            .header("X-Vault-Token", vaultConfig.token)
-            .send(backend)
-        }) { response =>
-          log.debug("response from vault: {}", response)
-          parseLease(response).map(renewed => renewed.copy(data = lease.data))
+      for {
+        now <- ReaderT.liftF(clock.realTime(TimeUnit.SECONDS))
+        response <- ReaderT[Effect, VaultConfig, Response[Either[String, String]]] { vaultConfig =>
+          sync.delay {
+            basicRequest
+              .post(uri"${vaultConfig.apiUrl}/sys/leases/renew")
+              .body(
+                Json
+                  .obj(
+                    "lease_id" -> Json.fromString(leaseId),
+                    "increment" -> Json.fromLong(increment.getOrElse(3600))
+                  )
+                  .noSpaces
+              )
+              .header("X-Vault-Token", vaultConfig.token)
+              .send(backend)
+          }
         }
-      }
+        lease <- {
+          ReaderT.liftF {
+            log.debug("response from vault: {}", response)
+            parseLease(response).map { renewed =>
+              renewed.copy(
+                data = lease.data,
+                creationTime = lease.creationTime,
+                lastRenewalTime = Some(now)
+              )
+            }
+          }
+        }
+      } yield lease
     }
     .getOrElse(
       ReaderT[Effect, VaultConfig, Lease](_ =>
         sync.raiseError(new IllegalArgumentException("no lease id defined"))
       )
     )
 
-  /** Auto-refresh a lease reference based on the current time.
+  /** Refresh a lease reference based on the current time.
     *
     * This is not a continuous refresh, the flow is:
     *
-    *   1. if lease is not renewable or is expired: create a new lease 2. if its
-    *      not expired but the current time is greater then issue time * refresh
-    *      ratio: refresh the current lease 3. return the current lease
-    *      otherwise
+    *   1. if lease is not renewable: create a new lease
+    *   2. if its not expired but the current time is greater then issue time * refresh ratio:
+    *      refresh the current lease
+    *   3. return the current lease otherwise
     *
     * @param leaseRef
     *   a reference to the current (maybe empty) lease
@@ -91,33 +129,34 @@ abstract class SecretEngineProvider[Effect[_], ProviderContext](implicit
     * @return
     *   an effect reader with the logic above applied
     */
-  def autoRefresh(
-      leaseRef: Ref[Effect, Option[Lease]],
-      increment: Option[Long] = None,
-      refreshRatio: Double = 0.5,
-      forceNew: Boolean = false
-  )(
-      newLease: ReaderT[Effect, VaultConfig, Lease]
+  def refresh(leaseRef: Ref[Effect, Option[Lease]],
+              refreshRatio: Double = 0.5)(
+    create: ReaderT[Effect, VaultConfig, Lease],
+    renew: Lease => ReaderT[Effect, VaultConfig, Lease],
+    maxTtlSeconds: ReaderT[Effect, VaultConfig, Long] = ReaderT.liftF(sync.pure(3600)),
+    maxReached: ReaderT[Effect, VaultConfig, Unit] = ReaderT.liftF(sync.unit)
   ): ReaderT[Effect, VaultConfig, Lease] = {
     for {
       now <- ReaderT.liftF(clock.realTime(TimeUnit.SECONDS))
       currentLease <- ReaderT.liftF(leaseRef.get)
+      max <- maxTtlSeconds
       valid <- {
         currentLease match {
           case Some(lease) =>
             val duration = lease.leaseDuration.getOrElse(0L)
-            val expiryTime = lease.issueTime + duration
+            val expiryTime = lease.lastRenewalTime.getOrElse(lease.creationTime) + duration
             val ratioTime = expiryTime - duration * refreshRatio
+            val totalDuration = lease.totalLeaseDuration(now)
 
-            if (!lease.renewable || now >= expiryTime) {
-              newLease
-            } else if (now >= ratioTime && !forceNew) {
-              renew(lease, increment)
-            } else if (now >= ratioTime && forceNew) {
-              newLease
+            if (!lease.renewable) {
+              create
+            } else if (totalDuration >= max) {
+              maxReached.flatMapF(_ => sync.pure(lease))
+            } else if (now >= ratioTime) {
+              renew(lease)
             } else ReaderT[Effect, VaultConfig, Lease](_ => sync.pure(lease))
 
-          case None => newLease
+          case None => create
         }
       }
       updated <- ReaderT[Effect, VaultConfig, Lease](_ =>
@@ -145,7 +184,7 @@ abstract class SecretEngineProvider[Effect[_], ProviderContext](implicit
       data = dto.data.getOrElse(Map.empty),
       renewable = dto.renewable,
       leaseDuration = dto.lease_duration,
-      issueTime = now
+      creationTime = now
     )
   }
 }

src/main/scala/com/drobisch/tresor/vault/package.scala

@@ -1,6 +1,7 @@
 package com.drobisch.tresor
 
-/** basic types to implement secrets coming from https://www.vaultproject.io
+/**
+  * basic types to implement secrets coming from https://www.vaultproject.io
   */
 package object vault {
   final case class VaultConfig(apiUrl: String, token: String)
@@ -17,14 +18,21 @@ package object vault {
     *   true if the lease validation period can be extends
     * @param leaseDuration
     *   duration of the lease starting with creation
+    * @param creationTime
+    *   the time of creation of the lease
+    * @param totalLeaseDuration
+    *   the total time of the lease
     */
   final case class Lease(
-      leaseId: Option[String],
-      data: Map[String, Option[String]],
-      renewable: Boolean,
-      leaseDuration: Option[Long],
-      issueTime: Long
-  )
+                          leaseId: Option[String],
+                          data: Map[String, Option[String]],
+                          renewable: Boolean,
+                          leaseDuration: Option[Long],
+                          creationTime: Long,
+                          lastRenewalTime: Option[Long] = None
+  ) {
+    def totalLeaseDuration(now: Long): Long = now - creationTime
+  }
 
   implicit object VaultSecretLease extends Secret[Lease] {
     override def data(secret: Lease): Option[Map[String, Option[String]]] =
@@ -34,7 +42,7 @@ package object vault {
     override def validDuration(secret: Lease): Option[Long] =
       secret.leaseDuration
     override def creationTime(secret: Lease): Option[Long] = Some(
-      secret.issueTime
+      secret.creationTime
     )
   }
 }