Add columns and file for daily billable usage (#276)

- Add the columns Report Start Time, Report End Time, Generated At
- Uploads the file to the daily location "NERC Storage <YYYY-MM-DD>.csv"

Author: knikolla

src/coldfront_plugin_cloud/management/commands/calculate_storage_gb_hours.py

@@ -1,7 +1,7 @@
 import csv
 from decimal import Decimal, ROUND_HALF_UP
 import dataclasses
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 import logging
 import os
 
@@ -38,6 +38,8 @@ def get_rates():
 @dataclasses.dataclass
 class InvoiceRow:
     InvoiceMonth: str = ""
+    Report_Start_Time: str = ""
+    Report_End_Time: str = ""
     Project_Name: str = ""
     Project_ID: str = ""
     PI: str = ""
@@ -50,12 +52,15 @@ class InvoiceRow:
     Invoice_Type: str = ""
     Rate: Decimal = 0
     Cost: Decimal = 0
+    Generated_At: str = ""
 
     @classmethod
     def get_headers(cls):
         """Returns all headers for display."""
         return [
             "Invoice Month",
+            "Report Start Time",
+            "Report End Time",
             "Project - Allocation",
             "Project - Allocation ID",
             "Manager (PI)",
@@ -68,6 +73,7 @@ def get_headers(cls):
             "SU Type",
             "Rate",
             "Cost",
+            "Generated At",
         ]
 
     def get_value(self, field: str):
@@ -156,7 +162,7 @@ def default_end_argument():
         return pytz.utc.localize(d)
 
     @staticmethod
-    def upload_to_s3(s3_endpoint, s3_bucket, file_location, invoice_month):
+    def upload_to_s3(s3_endpoint, s3_bucket, file_location, invoice_month, end_time):
         s3_key_id = os.getenv("S3_INVOICING_ACCESS_KEY_ID")
         s3_secret = os.getenv("S3_INVOICING_SECRET_ACCESS_KEY")
 
@@ -182,7 +188,18 @@ def upload_to_s3(s3_endpoint, s3_bucket, file_location, invoice_month):
         s3.upload_file(file_location, Bucket=s3_bucket, Key=primary_location)
         logger.info(f"Uploaded to {primary_location}.")
 
-        timestamp = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")
+        # Upload daily copy
+        # End time is exclusive, subtract one second to find the inclusive end date
+        invoice_date = end_time - timedelta(seconds=1)
+        invoice_date = invoice_date.strftime("%Y-%m-%d")
+        daily_location = (
+            f"Invoices/{invoice_month}/Service Invoices/NERC Storage {invoice_date}.csv"
+        )
+        s3.upload_file(file_location, Bucket=s3_bucket, Key=daily_location)
+        logger.info(f"Uploaded to {daily_location}.")
+
+        # Archival copy
+        timestamp = datetime.now(tz=timezone.utc).strftime("%Y%m%dT%H%M%SZ")
         secondary_location = (
             f"Invoices/{invoice_month}/"
             f"Archive/NERC Storage {invoice_month} {timestamp}.csv"
@@ -191,6 +208,8 @@ def upload_to_s3(s3_endpoint, s3_bucket, file_location, invoice_month):
         logger.info(f"Uploaded to {secondary_location}.")
 
     def handle(self, *args, **options):
+        generated_at = datetime.now(tz=timezone.utc).isoformat(timespec="seconds")
+
         def get_outages_for_service(cluster_name: str):
             """Get outages for a service from nerc-rates.
 
@@ -220,6 +239,8 @@ def process_invoice_row(allocation, attrs, su_name, rate):
             if time > 0:
                 row = InvoiceRow(
                     InvoiceMonth=options["invoice_month"],
+                    Report_Start_Time=options["start"].isoformat(),
+                    Report_End_Time=options["end"].isoformat(),
                     Project_Name=allocation.get_attribute(
                         attributes.ALLOCATION_PROJECT_NAME
                     ),
@@ -236,6 +257,7 @@ def process_invoice_row(allocation, attrs, su_name, rate):
                     Invoice_Type=su_name,
                     Rate=rate,
                     Cost=(time * rate).quantize(Decimal(".01"), rounding=ROUND_HALF_UP),
+                    Generated_At=generated_at,
                 )
                 csv_invoice_writer.writerow(row.get_values())
 
@@ -331,4 +353,5 @@ def process_invoice_row(allocation, attrs, su_name, rate):
                 options["s3_bucket_name"],
                 options["output"],
                 options["invoice_month"],
+                options["end"],
             )