A way to record skipped files in prometheus

Author: stephen-riggs

src/murfey/client/multigrid_control.py

@@ -607,8 +607,15 @@ def _increment_transferred_files_prometheus(
             requests.post(url, json=data)
 
     def _increment_transferred_files(
-        self, updates: List[RSyncerUpdate], source: str, destination: str
+        self,
+        updates: List[RSyncerUpdate],
+        num_skipped_files: int,
+        source: str,
+        destination: str,
     ):
+        skip_url = f"{str(self._environment.url.geturl())}{url_path_for('prometheus.router', 'increment_rsync_skipped_files_prometheus', visit_name=self._environment.visit, increment_count=num_skipped_files)}"
+        requests.post(skip_url, json={})
+
         checked_updates = [
             update for update in updates if update.outcome is TransferResult.SUCCESS
         ]

src/murfey/client/rsync.py

@@ -309,7 +309,7 @@ def _fake_transfer(self, files: list[Path]) -> bool:
             self.notify(update)
             updates.append(update)
             time.sleep(0.01)
-            self.notify([update], secondary=True)
+            self.notify([update], num_skipped_files=0, secondary=True)
         # self.notify(updates, secondary=True)
 
         return True
@@ -328,8 +328,10 @@ def _transfer(self, infiles: list[Path]) -> bool:
                 if f.is_file() and f.stat().st_ctime < self._end_time.timestamp()
             ]
             self._skipped_files.extend(set(infiles).difference(set(files)))
+            num_skipped_files = len(set(infiles).difference(set(files)))
         else:
             files = [f for f in infiles if f.is_file()]
+            num_skipped_files = 0
 
         previously_transferred = self._files_transferred
         transfer_success: set[Path] = set()
@@ -528,7 +530,9 @@ def parse_stderr(line: str):
             if success:
                 success = result.returncode == 0
 
-        self.notify(successful_updates, secondary=True)
+        self.notify(
+            successful_updates, num_skipped_files=num_skipped_files, secondary=True
+        )
 
         # Print out a summary message for each file transfer batch instead of individual messages
         # List out file paths as stored in memory to see if issue is due to file path mismatch

src/murfey/server/api/prometheus.py

@@ -90,6 +90,15 @@ def increment_rsync_transferred_files_prometheus(
     ).inc(rsyncer_info.data_bytes)
 
 
+@router.post(
+    "/visits/{visit_name}/increment_rsync_skipped_files_prometheus/{increment_count}"
+)
+def increment_rsync_skipped_files_prometheus(
+    visit_name: str, increment_count: int, db=murfey_db
+):
+    prom.skipped_files.labels(visit=visit_name).inc(increment_count)
+
+
 @router.post("/visits/{visit_name}/monitoring/{on}")
 def change_monitoring_status(visit_name: str, on: int):
     prom.monitoring_switch.labels(visit=visit_name)

src/murfey/server/prometheus.py

@@ -24,6 +24,12 @@
     ["rsync_source", "visit"],
 )
 
+skipped_files = Gauge(
+    "skipped_files",
+    "Number of files not transferred due to end time",
+    ["visit"],
+)
+
 preprocessed_movies = Counter(
     "preprocessed_movies",
     "Number of movies that have been preprocessed",

src/murfey/util/route_manifest.yaml

@@ -642,6 +642,13 @@ murfey.server.api.prometheus.router:
         type: str
     methods:
       - POST
+  - path: /prometheus/visits/{visit_name}/increment_rsync_skipped_files_prometheus/{increment_count}
+    function: increment_rsync_skipped_files_prometheus
+    path_params:
+      - name: visit_name
+        type: str
+      - name: increment_count
+        type: int
   - path: /prometheus/visits/{visit_name}/monitoring/{on}
     function: change_monitoring_status
     path_params: