lib: fix virtual package lookup performance

Instead of having a dict with pkgvers as keys and iterating over all
keys to match virtual packages, change the map to have pkgnames as keys,
which hold another dict with all the old keys and values for a given
package name.

This significantly speeds up XBPS, since for every package it would
iterate over the whole virtual package map. With recent changes to
xbps-src, where it generates cmd: and py3: virtual packages,
the number of virtual packages in the map has been growing a lot.

Author: Duncaen

include/xbps_api_impl.h

@@ -113,7 +113,7 @@ int HIDDEN xbps_register_pkg(struct xbps_handle *, xbps_dictionary_t);
 char HIDDEN *xbps_archive_get_file(struct archive *, struct archive_entry *);
 xbps_dictionary_t HIDDEN xbps_archive_get_dictionary(struct archive *,
 		struct archive_entry *);
-const char HIDDEN *vpkg_user_conf(struct xbps_handle *, const char *, bool);
+const char HIDDEN *vpkg_user_conf(struct xbps_handle *, const char *);
 xbps_array_t HIDDEN xbps_get_pkg_fulldeptree(struct xbps_handle *,
 		const char *, bool);
 struct xbps_repo HIDDEN *xbps_regget_repo(struct xbps_handle *,

lib/conf.c

@@ -51,10 +51,47 @@
  * Functions for parsing xbps configuration files.
  */
 
+static int
+vpkg_map_add(xbps_dictionary_t d, const char *pkgname, const char *vpkgver, const char *provider)
+{
+	xbps_dictionary_t providers;
+	bool alloc = false;
+	int r;
+
+	providers = xbps_dictionary_get(d, pkgname);
+	if (!providers) {
+		providers = xbps_dictionary_create();
+		if (!providers)
+			return -ENOMEM;
+		alloc = true;
+		if (!xbps_dictionary_set(d, pkgname, providers)) {
+			r = errno ? -errno : -ENOMEM;
+			goto err;
+		}
+	}
+
+	if (!xbps_dictionary_set_cstring(providers, vpkgver, provider)) {
+		r = errno ? -errno : -ENOMEM;
+		goto err;
+	}
+
+	r = 0;
+err:
+	if (alloc)
+		xbps_object_release(providers);
+	return r;
+}
+
 static int
 store_virtualpkg(struct xbps_handle *xhp, const char *path, size_t line, char *val)
 {
+	char namebuf[XBPS_NAME_SIZE];
+	char pkgverbuf[XBPS_NAME_SIZE + sizeof("-99999_1")];
+	const char *vpkgname, *vpkgver, *provider;
 	char *p;
+	int r;
+
+
 	/*
 	 * Parse strings delimited by ':' i.e
 	 * 	<left>:<right>
@@ -66,13 +103,25 @@ store_virtualpkg(struct xbps_handle *xhp, const char *path, size_t line, char *v
 		return 0;
 	}
 	*p++ = '\0';
+	provider = p;
+
+	if (xbps_pkg_name(namebuf, sizeof(namebuf), val)) {
+		vpkgname = namebuf;
+		vpkgver = val;
+	} else {
+		vpkgname = val;
+		snprintf(pkgverbuf, sizeof(pkgverbuf), "%s-99999_1", vpkgname);
+		vpkgver = pkgverbuf;
+	}
 
-	if (!xbps_dictionary_set_cstring(xhp->vpkgd, val, p))
-		return -errno;
-	if (!xbps_dictionary_set_cstring(xhp->vpkgd_conf, val, p))
-		return -errno;
+	r = vpkg_map_add(xhp->vpkgd, vpkgname, vpkgver, provider);
+	if (r < 0)
+		return r;
+	r = vpkg_map_add(xhp->vpkgd_conf, vpkgname, vpkgver, provider);
+	if (r < 0)
+		return r;
 	xbps_dbg_printf("%s: added virtualpkg %s for %s\n", path, val, p);
-	return 1;
+	return 0;
 }
 
 static void

lib/pkgdb.c

@@ -35,6 +35,7 @@
 #include <string.h>
 #include <unistd.h>
 
+#include "xbps.h"
 #include "xbps_api_impl.h"
 
 /**
@@ -145,26 +146,35 @@ pkgdb_map_vpkgs(struct xbps_handle *xhp)
 {
 	xbps_object_iterator_t iter;
 	xbps_object_t obj;
-	int rv = 0;
+	int r = 0;
 
 	if (!xbps_dictionary_count(xhp->pkgdb))
 		return 0;
 
 	if (xhp->vpkgd == NULL) {
 		xhp->vpkgd = xbps_dictionary_create();
-		assert(xhp->vpkgd);
+		if (!xhp->vpkgd) {
+			r = -errno;
+			xbps_error_printf("failed to create dictionary\n");
+			return r;
+		}
 	}
+
 	/*
 	 * This maps all pkgs that have virtualpkgs in pkgdb.
 	 */
 	iter = xbps_dictionary_iterator(xhp->pkgdb);
-	assert(iter);
+	if (!iter) {
+		r = -errno;
+		xbps_error_printf("failed to create iterator");
+		goto out;
+	}
 
 	while ((obj = xbps_object_iterator_next(iter))) {
 		xbps_array_t provides;
 		xbps_dictionary_t pkgd;
 		const char *pkgver = NULL;
-		char pkgname[XBPS_NAME_SIZE] = {0};
+		const char *pkgname = NULL;
 		unsigned int cnt;
 
 		pkgd = xbps_dictionary_get_keysym(xhp->pkgdb, obj);
@@ -174,26 +184,53 @@ pkgdb_map_vpkgs(struct xbps_handle *xhp)
 			continue;
 
 		xbps_dictionary_get_cstring_nocopy(pkgd, "pkgver", &pkgver);
-		if (!xbps_pkg_name(pkgname, sizeof(pkgname), pkgver)) {
-			rv = EINVAL;
-			goto out;
-		}
+		xbps_dictionary_get_cstring_nocopy(pkgd, "pkgname", &pkgname);
+		assert(pkgname);
+
 		for (unsigned int i = 0; i < cnt; i++) {
+			char vpkgname[XBPS_NAME_SIZE];
 			const char *vpkg = NULL;
+			xbps_dictionary_t providers;
+			bool alloc = false;
 
 			xbps_array_get_cstring_nocopy(provides, i, &vpkg);
-			if (!xbps_dictionary_set_cstring(xhp->vpkgd, vpkg, pkgname)) {
-				xbps_dbg_printf("%s: set_cstring vpkg "
-				    "%s pkgname %s\n", __func__, vpkg, pkgname);
-				rv = EINVAL;
+			if (!xbps_pkg_name(vpkgname, sizeof(vpkgname), vpkg)) {
+				xbps_warn_printf("%s: invalid provides: %s\n", pkgver, vpkg);
+				continue;
+			}
+
+			providers = xbps_dictionary_get(xhp->vpkgd, vpkgname);
+			if (!providers) {
+				providers = xbps_dictionary_create();
+				if (!providers) {
+					r = -errno;
+					xbps_error_printf("failed to create dictionary\n");
+					goto out;
+				}
+				if (!xbps_dictionary_set(xhp->vpkgd, vpkgname, providers)) {
+					r = -errno;
+					xbps_error_printf("failed to set dictionary entry\n");
+					xbps_object_release(providers);
+					goto out;
+				}
+				alloc = true;
+			}
+
+			if (!xbps_dictionary_set_cstring(providers, vpkg, pkgname)) {
+				r = -errno;
+				xbps_error_printf("failed to set dictionary entry\n");
+				if (alloc)
+					xbps_object_release(providers);
 				goto out;
 			}
+			if (alloc)
+				xbps_object_release(providers);
 			xbps_dbg_printf("[pkgdb] added vpkg %s for %s\n", vpkg, pkgname);
 		}
 	}
 out:
 	xbps_object_iterator_release(iter);
-	return rv;
+	return r;
 }
 
 static int
@@ -398,13 +435,17 @@ generate_full_revdeps_tree(struct xbps_handle *xhp)
 {
 	xbps_object_t obj;
 	xbps_object_iterator_t iter;
+	xbps_dictionary_t vpkg_cache;
 
 	if (xhp->pkgdb_revdeps)
 		return;
 
 	xhp->pkgdb_revdeps = xbps_dictionary_create();
 	assert(xhp->pkgdb_revdeps);
 
+	vpkg_cache = xbps_dictionary_create();
+	assert(vpkg_cache);
+
 	iter = xbps_dictionary_iterator(xhp->pkgdb);
 	assert(iter);
 
@@ -421,20 +462,33 @@ generate_full_revdeps_tree(struct xbps_handle *xhp)
 		xbps_dictionary_get_cstring_nocopy(pkgd, "pkgver", &pkgver);
 		for (unsigned int i = 0; i < xbps_array_count(rundeps); i++) {
 			xbps_array_t pkg;
-			const char *pkgdep = NULL, *vpkgname = NULL;
-			char *v, curpkgname[XBPS_NAME_SIZE];
+			const char *pkgdep = NULL, *v;
+			char curpkgname[XBPS_NAME_SIZE];
 			bool alloc = false;
 
 			xbps_array_get_cstring_nocopy(rundeps, i, &pkgdep);
 			if ((!xbps_pkgpattern_name(curpkgname, sizeof(curpkgname), pkgdep)) &&
 			    (!xbps_pkg_name(curpkgname, sizeof(curpkgname), pkgdep))) {
 					abort();
 			}
-			vpkgname = vpkg_user_conf(xhp, curpkgname, false);
-			if (vpkgname == NULL) {
-				v = strdup(curpkgname);
-			} else {
-				v = strdup(vpkgname);
+
+			/* TODO: this is kind of a workaround, to avoid calling vpkg_user_conf
+			 * over and over again for the same packages which is slow. A better
+			 * solution for itself vpkg_user_conf being slow should probably be
+			 * implemented at some point.
+			 */
+			if (!xbps_dictionary_get_cstring_nocopy(vpkg_cache, curpkgname, &v)) {
+				const char *vpkgname = vpkg_user_conf(xhp, curpkgname);
+				if (vpkgname) {
+					v = vpkgname;
+				} else {
+					v = curpkgname;
+				}
+				errno = 0;
+				if (!xbps_dictionary_set_cstring_nocopy(vpkg_cache, curpkgname, v)) {
+					xbps_error_printf("%s\n", strerror(errno ? errno : ENOMEM));
+					abort();
+				}
 			}
 
 			pkg = xbps_dictionary_get(xhp->pkgdb_revdeps, v);
@@ -446,12 +500,12 @@ generate_full_revdeps_tree(struct xbps_handle *xhp)
 				xbps_array_add_cstring_nocopy(pkg, pkgver);
 				xbps_dictionary_set(xhp->pkgdb_revdeps, v, pkg);
 			}
-			free(v);
 			if (alloc)
 				xbps_object_release(pkg);
 		}
 	}
 	xbps_object_iterator_release(iter);
+	xbps_object_release(vpkg_cache);
 }
 
 xbps_array_t