Add simplified VEXT loading as -E<name>

bin/varnishd/Makefile.am

@@ -63,7 +63,6 @@ varnishd_SOURCES = \
 	cache/cache_ws_common.c \
 	common/common_vsc.c \
 	common/common_vsmw.c \
-	common/common_vext.c \
 	hash/hash_classic.c \
 	hash/hash_critbit.c \
 	hash/hash_simple_list.c \
@@ -99,6 +98,7 @@ varnishd_SOURCES = \
 	mgt/mgt_util.c \
 	mgt/mgt_vcc.c \
 	mgt/mgt_vcl.c \
+	mgt/mgt_vext.c \
 	proxy/cache_proxy_proto.c \
 	proxy/cache_proxy.h \
 	storage/mgt_stevedore.c \

bin/varnishd/common/heritage.h

@@ -134,12 +134,3 @@ extern vsm_lock_f *vsc_lock;
 extern vsm_lock_f *vsc_unlock;
 extern vsm_lock_f *vsmw_lock;
 extern vsm_lock_f *vsmw_unlock;
-
-/* common/common_vext.c */
-
-void vext_argument(const char *);
-void vext_copyin(struct vsb *);
-void vext_load(void);
-void vext_cleanup(int);
-typedef void vext_iter_f(const char *, void *);
-void vext_iter(vext_iter_f *func, void *);

bin/varnishd/mgt/mgt.h

@@ -263,3 +263,11 @@ pthread_create(void)
 #define MGT_EXPERIMENT(x)	COM_EXPERIMENT(mgt_param.experimental_bits, x)
 #define MGT_DO_DEBUG(x)		COM_DO_DEBUG(mgt_param.debug_bits, x)
 #define MGT_VCC_FEATURE(x)	COM_VCC_FEATURE(mgt_param.vcc_feature_bits, x)
+
+/* mgt_vext.c */
+void vext_argument(const char *);
+void vext_copyin(struct vsb *);
+void vext_load(void);
+void vext_cleanup(int);
+typedef void vext_iter_f(const char *, void *);
+void vext_iter(vext_iter_f *func, void *);

bin/varnishd/common/common_vext.c → bin/varnishd/mgt/mgt_vext.c

@@ -40,15 +40,17 @@
 #include <string.h>
 #include <unistd.h>
 
-#include "vdef.h"
+#include "mgt.h"
+
 #include "vas.h"
 #include "miniobj.h"
 #include "vav.h"
+#include "vfil.h"
 #include "vqueue.h"
 #include "vrnd.h"
 #include "vsb.h"
 
-#include "heritage.h"
+#include "common/heritage.h"
 
 struct vext {
 	unsigned		magic;
@@ -64,10 +66,25 @@ struct vext {
 static VTAILQ_HEAD(,vext) vext_list =
     VTAILQ_HEAD_INITIALIZER(vext_list);
 
+static int
+vext_tryopen(void *priv, const char *fn)
+{
+	struct vext *vp;
+
+	CAST_OBJ_NOTNULL(vp, priv, VEXT_MAGIC);
+	vp->fd = open(fn, O_RDONLY);
+	if (vp->fd < 0)
+		return (1);
+	return (0);
+}
+
 void
 vext_argument(const char *arg)
 {
 	struct vext *vp;
+	char fn[1024];
+	char *fno = NULL;
+	struct vfil_path *vmod_path = NULL;
 
 	fprintf(stderr, "EEE <%s>\n", arg);
 	ALLOC_OBJ(vp, VEXT_MAGIC);
@@ -78,11 +95,20 @@ vext_argument(const char *arg)
 		ARGV_ERR("\tParse failure in argument: %s\n\t%s\n",
 		    arg, vp->argv[0]);
 	VTAILQ_INSERT_TAIL(&vext_list, vp, list);
-	fprintf(stderr, "eee <%s>\n", vp->argv[1]);
-	vp->fd = open(vp->argv[1], O_RDONLY);
-	if (vp->fd < 0)
+
+	if (strchr(vp->argv[1], '/'))
+		bstrcpy(fn, vp->argv[1]);
+	else
+		bprintf(fn, "libvmod_%s.so", vp->argv[1]);
+
+	VFIL_setpath(&vmod_path, mgt_vmod_path);
+	if (VFIL_searchpath(vmod_path, vext_tryopen, vp, fn, &fno)) {
 		ARGV_ERR("\tCannot open %s\n\t%s\n",
-		    vp->argv[1], strerror(errno));
+		    fn, strerror(errno));
+	}
+
+	fprintf(stderr, "eee <%s>\n", fno);
+	free(fno);
 }
 
 void
@@ -95,7 +121,7 @@ vext_iter(vext_iter_f *func, void *priv)
 }
 
 void
-vext_copyin(struct vsb *vident)
+vext_copyin(struct vsb *vi)
 {
 	struct vext *vp;
 	const char *p;
@@ -114,8 +140,8 @@ vext_copyin(struct vsb *vident)
 		if (p != NULL)
 			p++;
 		else
-			p = vp->argv[0];
-		VSB_printf(vident, ",-E%s", p);
+			p = vp->argv[1];
+		VSB_printf(vi, ",-E%s", p);
 		VSB_printf(vp->vsb, "vext_cache/%s,", p);
 		for (i = 0; i < 8; i++) {
 			AZ(VRND_RandomCrypto(&u, sizeof u));

bin/varnishtest/tests/x00000.vtc

@@ -2,7 +2,7 @@ varnishtest "Test VMOD import from VEXT"
 
 feature topbuild
 
-server s1 {
+server s1 -repeat 2 {
 	rxreq
 	txresp
 } -start
@@ -23,3 +23,26 @@ client c1 {
 	rxresp
 	expect resp.http.foobar ~ [0-9]
 } -run
+
+varnish v1 -stop
+
+varnish v2 \
+	-arg "-Estd" \
+	-vcl+backend {
+		import std;
+
+		sub vcl_deliver {
+			set resp.http.foobar = std.random(10,99);
+		}
+	} -start
+
+client c2 -connect ${v2_sock} {
+	txreq
+	rxresp
+	expect resp.http.foobar ~ [0-9]
+} -run
+
+varnish v2 -stop
+
+shell -err -match {Error:.*Cannot open libvmod_std.so} \
+	"varnishd -pvmod_path=/nonexistent -Estd -b none -a ${tmpdir}/vtc.sock"

doc/sphinx/reference/varnishd.rst

@@ -25,6 +25,7 @@ varnishd
     [-b [host[:port]|path]]
     [-C]
     [-d]
+    [-E extension]
     [-F]
     [-f config]
     [-h type[,options]]
@@ -229,6 +230,22 @@ Operations options
   Execute the management commands in the file given as ``clifile``
   before the worker process starts, see `CLI Command File`_.
 
+-E extension
+
+  Load the named extension. Extensions are modules (VMODs) which can modify
+  varnishd behavior outside VCL, for example by adding storage- or protocol
+  implementations. Extensions are always also VMODs, even if they do not provide
+  any functionality to VCL, but in most cases they will.
+
+  Any VMOD can be loaded as an extension, in which case it is simply pre-loaded
+  at startup and prevented from ever getting unloaded.
+
+  If `extension` contains any slashes ``/``, it is taken as an abslute path.
+  Otherwise, for `extention` ``ext``, a ``vmod_ext.so`` is searched for in
+  ``vmod_path``.
+
+  ``varnishd`` startup fails if loading the extension fails for any reason.
+
 Tuning options
 --------------
 

tools/lsan.suppr

@@ -8,6 +8,7 @@ leak:HSH_config
 leak:vcc_
 leak:VSL_Setup
 leak:WRK_BgThread
+leak:vext_argument
 #
 leak:VBH_new
 # ev