Fixes to handle multi-file upload. (#40)

* Fixes to handle multi-file upload.

* Creates manifest when generating tar.gz with specified files
* Updates collect role to use package_reports output
* Removed upload files after upload

* Switch to a 3 day lookback window to reduce the size of the view.

Author: chambridge

.gitignore

@@ -6,3 +6,6 @@ molecule/test-local/.kube/
 
 # testing directory for local development to modify openshift resources
 testing/
+
+# vscode settings
+.vscode/

molecule/crds/report_crd.yaml

@@ -1,3 +1,4 @@
+---
 kind: CustomResourceDefinition
 apiVersion: apiextensions.k8s.io/v1beta1
 metadata:

molecule/crds/report_datasource_crd.yaml

@@ -1,3 +1,4 @@
+---
 kind: CustomResourceDefinition
 apiVersion: apiextensions.k8s.io/v1beta1
 metadata:

molecule/crds/report_query_crd.yaml

@@ -1,3 +1,4 @@
+---
 kind: CustomResourceDefinition
 apiVersion: apiextensions.k8s.io/v1beta1
 metadata:

roles/collect/files/package_report.py

@@ -26,7 +26,7 @@
 import sys
 import tarfile
 from datetime import datetime
-
+from uuid import uuid4
 
 DEFAULT_MAX_SIZE = 100
 MEGABYTE = 1024 * 1024
@@ -82,7 +82,6 @@ def parse_args():
                         help="OCP Cluster ID")
     parser.add_argument("-v", "--verbosity", action="count",
                         default=0, help="increase verbosity (up to -vvv)")
-    parser.add_argument("--manifest-id", required=True, help="Manifest UUID")
     return parser.parse_args()
 
 
@@ -179,19 +178,25 @@ def split_files(filepath, max_size):
             LOG.info(f"Split files: {split_files}")
 
 
-def render_manifest(args):
+def render_manifest(args, archivefiles=[]):
     """Render the manifest template and write it to a file.
 
     Args:
         args (Namespace) an ArgumentParser Namespace object
 
     Returns:
         (str) the rendered manifest file name
+        (str) the manifest uuid
     """
     manifest = TEMPLATE
+    manifest_uuid = str(uuid4())
     manifest["cluster_id"] = args.ocp_cluster_id
-    manifest["files"] = os.listdir(args.filepath)
-    manifest["uuid"] = args.manifest_id
+    manifest["uuid"] = manifest_uuid
+    manifest_files = []
+    for idx in range(len(archivefiles)): 
+        upload_name = f"{manifest_uuid}_openshift_usage_report.{idx}.csv"
+        manifest_files.append(upload_name)
+    manifest["files"] = manifest_files
     LOG.debug(f"rendered manifest: {manifest}")
     manifest_filename = f"{args.filepath}/manifest.json"
 
@@ -206,13 +211,14 @@ def render_manifest(args):
         LOG.critical(f"Fatal error: {exc}")
         sys.exit(2)
     LOG.info(f"manifest generated")
-    return manifest_filename
+    return (manifest_filename, manifest_uuid)
 
 
-def write_tarball(tarfilename, archivefiles=[]):
+def write_tarball(args, tarfilename, archivefiles=[]):
     """Write a tarball, adding the given files to the archive.
 
     Args:
+        args (Namespace) an ArgumentParser Namespace object
         tarfilename (str) the name of the tarball to create
         archivefiles (list) the list of files to include in the archive
 
@@ -222,11 +228,20 @@ def write_tarball(tarfilename, archivefiles=[]):
     Raises:
         FileExistsError if tarfilename already exists
     """
+    if not archivefiles:
+        return None
+    
+    manifest_filename, manifest_uuid = render_manifest(args, archivefiles)
     try:
         with tarfile.open(tarfilename, f"{FILE_FLAG}:gz") as tarball:
+            file_count = 0
             for fname in archivefiles:
                 LOG.debug(f"Adding {fname} to {tarfilename}: ")
-                tarball.add(fname, arcname=os.path.sep)
+                if fname.endswith(".csv"):
+                    upload_name = f"{manifest_uuid}_openshift_usage_report.{file_count}.csv"
+                    tarball.add(fname, arcname=upload_name)
+                    file_count += 1
+            tarball.add(manifest_filename, arcname="manifest.json")
     except FileExistsError as exc:
         LOG.critical(exc)
         sys.exit(2)
@@ -247,20 +262,22 @@ def write_tarball(tarfilename, archivefiles=[]):
     need_split = need_split(args.filepath, args.max_size)
     if need_split:
         split_files(args.filepath, args.max_size)
-        manifest_filename = render_manifest(args)
-
         tarpath = args.filepath + "/../"
         tarfiletmpl = "cost-mgmt{}.tar.gz"
         for idx, filename in enumerate(os.listdir(args.filepath)):
             if ".csv" in filename:
                 tarfilename = os.path.abspath(
                     tarpath + tarfiletmpl.format(idx))
-                out_files.append(write_tarball(
-                    tarfilename, [f"{args.filepath}/{filename}", manifest_filename]))
+                output_tar = write_tarball(args, 
+                    tarfilename, [f"{args.filepath}/{filename}"])
+                if output_tar:
+                    out_files.append(output_tar)
     else:
-        render_manifest(args)
         tarfilename = os.path.abspath(args.filepath + "/../cost-mgmt.tar.gz")
-        out_files.append(write_tarball(tarfilename, [args.filepath]))
+        files = [f"{args.filepath}/{filename}" for filename in os.listdir(args.filepath)]
+        output_tar = write_tarball(args, tarfilename, files)
+        if output_tar:
+            out_files.append(output_tar)
 
     for fname in out_files:
         print(fname)

roles/collect/tasks/main.yml

@@ -320,7 +320,7 @@
     - '{{ csv_stat_result.results }}'
 
 - name: Run packaging script to prepare reports for sending to Insights
-  script: package_report.py --filepath {{ ocp_cluster_id }} --max-size {{ collect_max_csvfile_size }} --ocp-cluster-id {{ ocp_cluster_id }} --overwrite --manifest-id {{ collect_file_prefix }}
+  script: package_report.py --filepath {{ ocp_cluster_id }} --max-size {{ collect_max_csvfile_size }} --ocp-cluster-id {{ ocp_cluster_id }} --overwrite
   args:
     chdir: '{{ collect_download_path }}'
   register: packaged_reports
@@ -336,12 +336,24 @@
 
 - name: Upload the cost report to ingress using basic auth
   shell:
-    cmd: 'curl -vvvv -F "file=@cost-mgmt.tar.gz;type=application/vnd.redhat.hccm.tar+tgz" {{ ingress_url }} -u {{ username }}:{{ password }} --cacert {{ cacert_path }}'
+    cmd: 'curl -vvvv -F "file=@{{ item }};type=application/vnd.redhat.hccm.tar+tgz" {{ ingress_url }} -u {{ username }}:{{ password }} --cacert {{ cacert_path }}'
     chdir: '{{ collect_download_path }}'
+  with_items:
+    - '{{ packaged_reports.stdout_lines }}'
   when: authentication == 'basic'
 
 - name: Upload the cost report to ingress using token auth
   shell:
-    cmd: 'curl -vvvv -F "file=@cost-mgmt.tar.gz;type=application/vnd.redhat.hccm.tar+tgz" {{ ingress_url }} -H "Authorization: Bearer {{ authentication_token }}" -H "User-Agent: cost-mgmt-operator/{{ source_commit }} cluster/{{ ocp_cluster_id }}" --cacert {{ cacert_path }}'
+    cmd: 'curl -vvvv -F "file=@{{ item }};type=application/vnd.redhat.hccm.tar+tgz" {{ ingress_url }} -H "Authorization: Bearer {{ authentication_token }}" -H "User-Agent: cost-mgmt-operator/{{ source_commit }} cluster/{{ ocp_cluster_id }}" --cacert {{ cacert_path }}'
     chdir: '{{ collect_download_path }}'
+  with_items:
+    - '{{ packaged_reports.stdout_lines }}'
   when: authentication == 'token'
+
+- name: Remove upload files
+  file:
+    path: '{{ collect_download_path }}/{{ item }}'
+    state: absent
+  with_items:
+    - '{{ packaged_reports.stdout_lines }}'
+  when: collect_delete_after | bool

roles/setup/files/cm_openshift_node_labels_lookback_report_query.yaml

@@ -40,4 +40,4 @@ spec:
       interval_end,
       node_labels
     FROM {| .Report.Inputs.CostManagementOpenShiftNodeLabelsReportName | reportTableName |}
-    WHERE {| .Report.Inputs.CostManagementOpenShiftNodeLabelsReportName | reportTableName |}.interval_start >= timestamp '{| default .Report.ReportingStart .Report.Inputs.ReportingStart | prestoTimestamp |}' - interval '10' day
+    WHERE {| .Report.Inputs.CostManagementOpenShiftNodeLabelsReportName | reportTableName |}.interval_start >= timestamp '{| default .Report.ReportingStart .Report.Inputs.ReportingStart | prestoTimestamp |}' - interval '3' day

roles/setup/files/cm_openshift_persistentvolumeclaim_lookback_report_query.yaml

@@ -75,4 +75,4 @@ spec:
       persistentvolume_labels,
       persistentvolumeclaim_labels
     FROM {| .Report.Inputs.PersistentVolumeClaimUsageReportName | reportTableName |}
-    WHERE {| .Report.Inputs.PersistentVolumeClaimUsageReportName | reportTableName |}.interval_start >= timestamp '{| default .Report.ReportingStart .Report.Inputs.ReportingStart | prestoTimestamp |}' - interval '10' day
+    WHERE {| .Report.Inputs.PersistentVolumeClaimUsageReportName | reportTableName |}.interval_start >= timestamp '{| default .Report.ReportingStart .Report.Inputs.ReportingStart | prestoTimestamp |}' - interval '3' day

roles/setup/files/cm_openshift_usage_lookback_report_query.yaml

@@ -91,4 +91,4 @@ spec:
       resource_id,
       pod_labels
     FROM {| .Report.Inputs.CostManagementOpenShiftUsageReportName | reportTableName |}
-    WHERE {| .Report.Inputs.CostManagementOpenShiftUsageReportName | reportTableName |}.interval_start >= timestamp '{| default .Report.ReportingStart .Report.Inputs.ReportingStart | prestoTimestamp |}' - interval '10' day
+    WHERE {| .Report.Inputs.CostManagementOpenShiftUsageReportName | reportTableName |}.interval_start >= timestamp '{| default .Report.ReportingStart .Report.Inputs.ReportingStart | prestoTimestamp |}' - interval '3' day