DAOS-16837 container: Add client-side DFS metrics (#15544)

If metrics are enabled for a POSIX container, create
a new pool/$UUID/container/$UUID/dfs metrics root in
the client telemetry to provide DFS-oriented metrics
(POSIX ops, file I/Os, etc).

Also fixes a bug in the agent code for pruning unused
client telemetry segments.

Required-githooks: true

Signed-off-by: Michael MacDonald <mjmac@google.com>

Author: mjmac

src/client/dfs/SConscript

@@ -44,7 +44,7 @@ def scons():
     libraries = ['daos_common', 'daos', 'uuid', 'gurt']
 
     dfs_src = ['common.c', 'cont.c', 'dir.c', 'file.c', 'io.c', 'lookup.c', 'mnt.c', 'obj.c',
-               'pipeline.c', 'readdir.c', 'rename.c', 'xattr.c', 'dfs_sys.c']
+               'pipeline.c', 'readdir.c', 'rename.c', 'xattr.c', 'dfs_sys.c', 'metrics.c']
     dfs = denv.d_library('dfs', dfs_src, LIBS=libraries)
     denv.Install('$PREFIX/lib64/', dfs)
 

src/client/dfs/common.c

@@ -625,6 +625,8 @@ entry_stat(dfs_t *dfs, daos_handle_t th, daos_handle_t oh, const char *name, siz
 		stbuf->st_atim.tv_sec  = stbuf->st_mtim.tv_sec;
 		stbuf->st_atim.tv_nsec = stbuf->st_mtim.tv_nsec;
 	}
+
+	DFS_OP_STAT_INCR(dfs, DOS_STAT);
 	return 0;
 }
 
@@ -710,6 +712,7 @@ open_dir(dfs_t *dfs, dfs_obj_t *parent, int flags, daos_oclass_id_t cid, struct
 			D_ASSERT(rc == 0);
 			dir->d.chunk_size = entry->chunk_size;
 			dir->d.oclass     = entry->oclass;
+			DFS_OP_STAT_INCR(dfs, DOS_MKDIR);
 			return 0;
 		}
 	}
@@ -742,6 +745,7 @@ open_dir(dfs_t *dfs, dfs_obj_t *parent, int flags, daos_oclass_id_t cid, struct
 	oid_cp(&dir->oid, entry->oid);
 	dir->d.chunk_size = entry->chunk_size;
 	dir->d.oclass     = entry->oclass;
+	DFS_OP_STAT_INCR(dfs, DOS_OPENDIR);
 	return 0;
 }
 

src/client/dfs/dfs_internal.h

@@ -15,6 +15,8 @@
 #include <daos.h>
 #include <daos_fs.h>
 
+#include "metrics.h"
+
 /** D-key name of SB metadata */
 #define SB_DKEY            "DFS_SB_METADATA"
 
@@ -105,6 +107,8 @@ typedef uint16_t dfs_layout_ver_t;
 
 /** object struct that is instantiated for a DFS open object */
 struct dfs_obj {
+	/** DFS mount point of object */
+	dfs_t        *dfs;
 	/** DAOS object ID */
 	daos_obj_id_t oid;
 	/** DAOS object open handle */
@@ -184,6 +188,8 @@ struct dfs {
 	struct dfs_mnt_hdls *cont_hdl;
 	/** the root dir stat buf */
 	struct stat          root_stbuf;
+	/** DFS top-level metrics */
+	struct dfs_metrics  *metrics;
 };
 
 struct dfs_entry {

src/client/dfs/dir.c

@@ -65,6 +65,7 @@ dfs_mkdir(dfs_t *dfs, dfs_obj_t *parent, const char *name, mode_t mode, daos_ocl
 	if (rc != 0)
 		return daos_der2errno(rc);
 
+	DFS_OP_STAT_INCR(dfs, DOS_MKDIR);
 	return rc;
 }
 
@@ -220,6 +221,7 @@ dfs_remove(dfs_t *dfs, dfs_obj_t *parent, const char *name, bool force, daos_obj
 	if (oid)
 		oid_cp(oid, entry.oid);
 
+	DFS_OP_STAT_INCR(dfs, DOS_UNLINK);
 out:
 	rc = check_tx(th, rc);
 	if (rc == ERESTART)

src/client/dfs/io.c

@@ -15,7 +15,20 @@
 
 #include "dfs_internal.h"
 
+static void
+dfs_update_file_metrics(dfs_t *dfs, daos_size_t read_bytes, daos_size_t write_bytes)
+{
+	if (dfs == NULL || dfs->metrics == NULL)
+		return;
+
+	if (read_bytes > 0)
+		d_tm_inc_gauge(dfs->metrics->dm_read_bytes, read_bytes);
+	if (write_bytes > 0)
+		d_tm_inc_gauge(dfs->metrics->dm_write_bytes, write_bytes);
+}
+
 struct dfs_read_params {
+	dfs_t           *dfs;
 	daos_size_t     *read_size;
 	daos_array_iod_t arr_iod;
 	daos_range_t     rg;
@@ -35,6 +48,8 @@ read_cb(tse_task_t *task, void *data)
 		D_GOTO(out, rc);
 	}
 
+	DFS_OP_STAT_INCR(params->dfs, DOS_READ);
+	dfs_update_file_metrics(params->dfs, params->arr_iod.arr_nr_read, 0);
 	*params->read_size = params->arr_iod.arr_nr_read;
 out:
 	D_FREE(params);
@@ -61,6 +76,7 @@ dfs_read_int(dfs_t *dfs, dfs_obj_t *obj, daos_off_t off, dfs_iod_t *iod, d_sg_li
 	if (params == NULL)
 		D_GOTO(err_task, rc = -DER_NOMEM);
 
+	params->dfs       = dfs;
 	params->read_size = read_size;
 
 	/** set array location */
@@ -90,6 +106,7 @@ dfs_read_int(dfs_t *dfs, dfs_obj_t *obj, daos_off_t off, dfs_iod_t *iod, d_sg_li
 	 * completion cb that frees params in this case, so we can just ignore the rc here.
 	 */
 	dc_task_schedule(task, true);
+
 	return 0;
 
 err_params:
@@ -125,6 +142,7 @@ dfs_read(dfs_t *dfs, dfs_obj_t *obj, d_sg_list_t *sgl, daos_off_t off, daos_size
 			daos_event_launch(ev);
 			daos_event_complete(ev, 0);
 		}
+		DFS_OP_STAT_INCR(dfs, DOS_READ);
 		return 0;
 	}
 
@@ -146,7 +164,9 @@ dfs_read(dfs_t *dfs, dfs_obj_t *obj, d_sg_list_t *sgl, daos_off_t off, daos_size
 			return daos_der2errno(rc);
 		}
 
+		DFS_OP_STAT_INCR(dfs, DOS_READ);
 		*read_size = iod.arr_nr_read;
+		dfs_update_file_metrics(dfs, iod.arr_nr_read, 0);
 		return 0;
 	}
 
@@ -173,6 +193,7 @@ dfs_readx(dfs_t *dfs, dfs_obj_t *obj, dfs_iod_t *iod, d_sg_list_t *sgl, daos_siz
 			daos_event_launch(ev);
 			daos_event_complete(ev, 0);
 		}
+		DFS_OP_STAT_INCR(dfs, DOS_READ);
 		return 0;
 	}
 
@@ -189,7 +210,9 @@ dfs_readx(dfs_t *dfs, dfs_obj_t *obj, dfs_iod_t *iod, d_sg_list_t *sgl, daos_siz
 			return daos_der2errno(rc);
 		}
 
+		DFS_OP_STAT_INCR(dfs, DOS_READ);
 		*read_size = arr_iod.arr_nr_read;
+		dfs_update_file_metrics(dfs, arr_iod.arr_nr_read, 0);
 		return 0;
 	}
 
@@ -223,6 +246,7 @@ dfs_write(dfs_t *dfs, dfs_obj_t *obj, d_sg_list_t *sgl, daos_off_t off, daos_eve
 			daos_event_launch(ev);
 			daos_event_complete(ev, 0);
 		}
+		DFS_OP_STAT_INCR(dfs, DOS_WRITE);
 		return 0;
 	}
 
@@ -238,8 +262,12 @@ dfs_write(dfs_t *dfs, dfs_obj_t *obj, d_sg_list_t *sgl, daos_off_t off, daos_eve
 		daos_event_errno_rc(ev);
 
 	rc = daos_array_write(obj->oh, DAOS_TX_NONE, &iod, sgl, ev);
-	if (rc)
+	if (rc == 0) {
+		DFS_OP_STAT_INCR(dfs, DOS_WRITE);
+		dfs_update_file_metrics(dfs, 0, buf_size);
+	} else {
 		D_ERROR("daos_array_write() failed, " DF_RC "\n", DP_RC(rc));
+	}
 
 	return daos_der2errno(rc);
 }
@@ -248,6 +276,8 @@ int
 dfs_writex(dfs_t *dfs, dfs_obj_t *obj, dfs_iod_t *iod, d_sg_list_t *sgl, daos_event_t *ev)
 {
 	daos_array_iod_t arr_iod;
+	daos_size_t      buf_size;
+	int              i;
 	int              rc;
 
 	if (dfs == NULL || !dfs->mounted)
@@ -266,6 +296,7 @@ dfs_writex(dfs_t *dfs, dfs_obj_t *obj, dfs_iod_t *iod, d_sg_list_t *sgl, daos_ev
 			daos_event_launch(ev);
 			daos_event_complete(ev, 0);
 		}
+		DFS_OP_STAT_INCR(dfs, DOS_WRITE);
 		return 0;
 	}
 
@@ -276,9 +307,18 @@ dfs_writex(dfs_t *dfs, dfs_obj_t *obj, dfs_iod_t *iod, d_sg_list_t *sgl, daos_ev
 	if (ev)
 		daos_event_errno_rc(ev);
 
+	buf_size = 0;
+	if (dfs->metrics != NULL && sgl != NULL)
+		for (i = 0; i < sgl->sg_nr; i++)
+			buf_size += sgl->sg_iovs[i].iov_len;
+
 	rc = daos_array_write(obj->oh, DAOS_TX_NONE, &arr_iod, sgl, ev);
-	if (rc)
+	if (rc == 0) {
+		DFS_OP_STAT_INCR(dfs, DOS_WRITE);
+		dfs_update_file_metrics(dfs, 0, buf_size);
+	} else {
 		D_ERROR("daos_array_write() failed (%d)\n", rc);
+	}
 
 	return daos_der2errno(rc);
 }

src/client/dfs/lookup.c

@@ -57,6 +57,7 @@ lookup_rel_path(dfs_t *dfs, dfs_obj_t *root, const char *path, int flags, dfs_ob
 	obj->d.oclass     = root->d.oclass;
 	obj->d.chunk_size = root->d.chunk_size;
 	obj->mode         = root->mode;
+	obj->dfs          = dfs;
 	strncpy(obj->name, root->name, DFS_MAX_NAME + 1);
 
 	rc = daos_obj_open(dfs->coh, obj->oid, daos_mode, &obj->oh, NULL);