Merge pull request #55 from knikolla/linting

Add pre-commit workflow

.github/workflows/pre-commit.yaml

@@ -0,0 +1,15 @@
+name: pre-commit
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v3
+    - uses: pre-commit/action@v3.0.1

.pre-commit-config.yaml

@@ -0,0 +1,16 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: trailing-whitespace
+      - id: check-merge-conflict
+      - id: end-of-file-fixer
+      - id: check-added-large-files
+      - id: check-case-conflict
+      - id: detect-private-key
+      - id: check-yaml
+
+  - repo: https://github.com/psf/black
+    rev: 24.4.0
+    hooks:
+      - id: black

src/openstack_billing_db/__init__.py

@@ -1 +0,0 @@
-

src/openstack_billing_db/billing.py

@@ -91,7 +91,7 @@ def collect_invoice_data_from_openstack(database, billing_start, billing_end, ra
             institution="",
             instances=project.instances,
             invoice_interval=f"{billing_start.date()} - {billing_end.date()}",
-            rates=rates
+            rates=rates,
         )
 
         for i in project.instances:  # type: model.Instance
@@ -126,8 +126,8 @@ def collect_invoice_data_from_openstack(database, billing_start, billing_end, ra
     return invoices
 
 
-def load_flavors_cache(flavors_cache_file) -> dict[int: model.Flavor]:
-    with open(flavors_cache_file, 'r') as f:
+def load_flavors_cache(flavors_cache_file) -> dict[int : model.Flavor]:
+    with open(flavors_cache_file, "r") as f:
         cache = json.load(f)
 
     flavors = []
@@ -138,12 +138,12 @@ def load_flavors_cache(flavors_cache_file) -> dict[int: model.Flavor]:
 
 
 def write_flavors_cache(flavors_cache_file, flavors):
-    with open(flavors_cache_file, 'w') as f:
+    with open(flavors_cache_file, "w") as f:
         f.write(json.dumps(flavors, indent=4))
 
 
 def merge_coldfront_data(invoices, coldfront_data_file):
-    with open(coldfront_data_file, 'r') as f:
+    with open(coldfront_data_file, "r") as f:
         allocations = json.load(f)
 
     by_project_id = {
@@ -155,16 +155,17 @@ def merge_coldfront_data(invoices, coldfront_data_file):
             a = by_project_id[invoice.project_id]
             invoice.project_name = a["attributes"]["Allocated Project Name"]
             invoice.institution_specific_code = a["attributes"].get(
-                "Institution-Specific Code", "N/A")
+                "Institution-Specific Code", "N/A"
+            )
             invoice.pi = a["project"]["pi"]
         except KeyError:
             continue
 
 
 def write(invoices, output, invoice_month=None):
-    with open(output, 'w', newline='') as f:
+    with open(output, "w", newline="") as f:
         csv_invoice_writer = csv.writer(
-            f, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL
+            f, delimiter=",", quotechar="|", quoting=csv.QUOTE_MINIMAL
         )
         # Write Headers
         csv_invoice_writer.writerow(
@@ -186,18 +187,26 @@ def write(invoices, output, invoice_month=None):
 
         for invoice in invoices:
             for invoice_type in [
-                'cpu', 'gpu_a100sxm4', 'gpu_a100', 'gpu_v100', 'gpu_k80', 'gpu_a2'
+                "cpu",
+                "gpu_a100sxm4",
+                "gpu_a100",
+                "gpu_v100",
+                "gpu_k80",
+                "gpu_a2",
             ]:
                 # Each project gets two rows, one for CPU and one for GPU
                 hours = invoice.__getattribute__(f"{invoice_type}_su_hours")
                 rate = invoice.rates.__getattribute__(invoice_type)
                 su_name = invoice.rates.__getattribute__(f"{invoice_type}_su_name")
                 cost = invoice.__getattribute__(f"{invoice_type}_su_cost")
                 if hours > 0:
-
                     csv_invoice_writer.writerow(
                         [
-                            invoice_month if invoice_month else invoice.invoice_interval,
+                            (
+                                invoice_month
+                                if invoice_month
+                                else invoice.invoice_interval
+                            ),
                             invoice.project_name,
                             invoice.project_id,
                             invoice.pi,
@@ -213,13 +222,17 @@ def write(invoices, output, invoice_month=None):
                     )
 
 
-def generate_billing(start, end, output, rates,
-                     coldfront_data_file=None,
-                     invoice_month=None,
-                     upload_to_s3=False,
-                     sql_dump_file=None,
-                     upload_to_primary_location=True):
-
+def generate_billing(
+    start,
+    end,
+    output,
+    rates,
+    coldfront_data_file=None,
+    invoice_month=None,
+    upload_to_s3=False,
+    sql_dump_file=None,
+    upload_to_primary_location=True,
+):
     database = model.Database(start, sql_dump_file)
 
     invoices = collect_invoice_data_from_openstack(database, start, end, rates)
@@ -228,15 +241,18 @@ def generate_billing(start, end, output, rates,
     write(invoices, output, invoice_month)
 
     if upload_to_s3:
-        s3_endpoint = os.getenv("S3_OUTPUT_ENDPOINT_URL",
-                                "https://s3.us-east-005.backblazeb2.com")
+        s3_endpoint = os.getenv(
+            "S3_OUTPUT_ENDPOINT_URL", "https://s3.us-east-005.backblazeb2.com"
+        )
         s3_bucket = os.getenv("S3_OUTPUT_BUCKET", "nerc-invoicing")
         s3_key_id = os.getenv("S3_OUTPUT_ACCESS_KEY_ID")
         s3_secret = os.getenv("S3_OUTPUT_SECRET_ACCESS_KEY")
 
         if not s3_key_id or not s3_secret:
-            raise Exception("Must provide S3_OUTPUT_ACCESS_KEY_ID and"
-                            " S3_OUTPUT_SECRET_ACCESS_KEY environment variables.")
+            raise Exception(
+                "Must provide S3_OUTPUT_ACCESS_KEY_ID and"
+                " S3_OUTPUT_SECRET_ACCESS_KEY environment variables."
+            )
         if not invoice_month:
             raise Exception("No invoice month specified. Required for S3 upload.")
 
@@ -255,7 +271,7 @@ def generate_billing(start, end, output, rates,
             s3.upload_file(output, Bucket=s3_bucket, Key=primary_location)
             logger.info(f"Uploaded to {primary_location}.")
 
-        timestamp = datetime.utcnow().strftime('%Y%m%dT%H%M%SZ')
+        timestamp = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")
         secondary_location = (
             f"Invoices/{invoice_month}/"
             f"Archive/NERC OpenStack {invoice_month} {timestamp}.csv"

src/openstack_billing_db/fetch.py

@@ -41,15 +41,18 @@ def download_latest_dump_from_s3() -> str:
     }
 
     """
-    s3_endpoint = os.getenv("S3_INPUT_ENDPOINT_URL",
-                            "https://holecs.rc.fas.harvard.edu")
+    s3_endpoint = os.getenv(
+        "S3_INPUT_ENDPOINT_URL", "https://holecs.rc.fas.harvard.edu"
+    )
     s3_bucket = os.getenv("S3_INPUT_BUCKET", "nerc-osp-backups")
     s3_key_id = os.getenv("S3_INPUT_ACCESS_KEY_ID")
     s3_secret = os.getenv("S3_INPUT_SECRET_ACCESS_KEY")
 
     if not s3_key_id or not s3_secret:
-        raise Exception("Must provide S3_INPUT_ACCESS_KEY_ID and"
-                        " S3_INPUT_SECRET_ACCESS_KEY environment variables.")
+        raise Exception(
+            "Must provide S3_INPUT_ACCESS_KEY_ID and"
+            " S3_INPUT_SECRET_ACCESS_KEY environment variables."
+        )
 
     s3 = boto3.client(
         "s3",
@@ -58,9 +61,8 @@ def download_latest_dump_from_s3() -> str:
         aws_secret_access_key=s3_secret,
     )
 
-    today = datetime.today().strftime('%Y%m%d')
-    dumps = s3.list_objects_v2(Bucket=s3_bucket,
-                               Prefix=f"dbs/nerc-ctl-0/nova-{today}")
+    today = datetime.today().strftime("%Y%m%d")
+    dumps = s3.list_objects_v2(Bucket=s3_bucket, Prefix=f"dbs/nerc-ctl-0/nova-{today}")
 
     if len(dumps["Contents"]) == 0:
         raise Exception(f"No database dumps found for {today}")
@@ -109,8 +111,10 @@ def convert_mysqldump_to_sqlite(path_to_dump) -> str:
         command = subprocess.run(["mysql2sqlite", path_to_dump], stdout=f)
 
     if command.returncode != 0:
-        raise Exception(f"Error converting {path_to_dump} to SQLite compatible"
-                        f" at {destination_path}.")
+        raise Exception(
+            f"Error converting {path_to_dump} to SQLite compatible"
+            f" at {destination_path}."
+        )
 
     logger.info(f"Converted at {destination_path}.")
     return destination_path
@@ -120,15 +124,19 @@ def get_keycloak_session():
     """Authenticate as a client with Keycloak to receive an access token."""
     keycloak_token_url = os.getenv(
         "KEYCLOAK_TOKEN_URL",
-        ("https://keycloak.mss.mghpcc.org/auth/realms/mss"
-         "/protocol/openid-connect/token")
+        (
+            "https://keycloak.mss.mghpcc.org/auth/realms/mss"
+            "/protocol/openid-connect/token"
+        ),
     )
     keycloak_client_id = os.getenv("KEYCLOAK_CLIENT_ID")
     keycloak_client_secret = os.getenv("KEYCLOAK_CLIENT_SECRET")
 
     if not keycloak_client_id or not keycloak_client_secret:
-        raise Exception("Must provide KEYCLOAK_CLIENT_ID and"
-                        " KEYCLOAK_CLIENT_SECRET environment variables.")
+        raise Exception(
+            "Must provide KEYCLOAK_CLIENT_ID and"
+            " KEYCLOAK_CLIENT_SECRET environment variables."
+        )
 
     r = requests.post(
         keycloak_token_url,
@@ -145,14 +153,14 @@ def get_keycloak_session():
     session.headers.update(headers)
     return session
 
+
 def download_coldfront_data(download_location=None) -> str:
     """Downloads allocation data from the ColdFront API.
 
     Returns location of downloaded JSON file.
     """
 
-    colfront_url = os.getenv("COLDFRONT_URL",
-                             "https://coldfront.mss.mghpcc.org")
+    colfront_url = os.getenv("COLDFRONT_URL", "https://coldfront.mss.mghpcc.org")
     allocations_url = f"{colfront_url}/api/allocations?all=true"
 
     logger.info(f"Making request to ColdFront at {allocations_url}")