[DNM] test pipeline failures

fence/__init__.py

@@ -395,12 +395,12 @@ def _load_keys(app, root_dir):
         root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 
     app.keypairs = keys.load_keypairs(os.path.join(root_dir, "keys"))
-
-    app.jwt_public_keys = {
+    pub_keys = {
         config["BASE_URL"]: OrderedDict(
             [(str(keypair.kid), str(keypair.public_key)) for keypair in app.keypairs]
         )
     }
+    app.jwt_public_keys = pub_keys
 
 
 def _setup_oidc_clients(app):

fence/blueprints/data/indexd.py

@@ -143,16 +143,20 @@ def get_signed_url_for_file(
     )
 
     prepare_presigned_url_audit_log(requested_protocol, indexed_file)
-    signed_url, authorized_user_from_passport = indexed_file.get_signed_url(
-        requested_protocol,
-        action,
-        expires_in,
-        force_signed_url=force_signed_url,
-        r_pays_project=r_pays_project,
-        file_name=file_name,
-        users_from_passports=users_from_passports,
-        bucket=bucket,
-    )
+    try:
+        signed_url, authorized_user_from_passport = indexed_file.get_signed_url(
+            requested_protocol,
+            action,
+            expires_in,
+            force_signed_url=force_signed_url,
+            r_pays_project=r_pays_project,
+            file_name=file_name,
+            users_from_passports=users_from_passports,
+            bucket=bucket,
+        )
+    except Exception as e:
+        logger.error("signed url error: ")
+        logger.error(str(e))
 
     # a single user from the list was authorized so update the audit log to reflect that
     # users info
@@ -574,7 +578,9 @@ def get_signed_url(
                 )
             # don't check the authorization if the file is public
             # (downloading public files with no auth is fine)
-            if not self.public_acl and not self.check_legacy_authorization(action):
+            not_a_public_acl = not self.public_acl
+            legacy_auth_failed = not self.check_legacy_authorization(action)
+            if not_a_public_acl and legacy_auth_failed:
                 raise Unauthorized(
                     f"You don't have access permission on this file: {self.file_id}"
                 )

fence/jwt/keys.py

@@ -11,10 +11,9 @@
         return default private key for the app
 """
 
-import datetime
+import logging
 import os
 
-import base64
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import serialization
 import dateutil.parser
@@ -78,6 +77,10 @@ def timestamp_key(name):
     keypairs = [
         Keypair.from_directory(d) for d in keypair_directories if os.path.isdir(d)
     ]
+    print("AAAAA")
+    print(keypairs)
+    logging.info("AAAA")
+    logging.info(keypairs)
 
     if not keypairs:
         raise EnvironmentError(

fence/jwt/token.py

@@ -179,7 +179,7 @@ def generate_signed_session_token(kid, private_key, expires_in, context=None):
     claims = {
         "pur": "session",
         "aud": ["fence", issuer],
-        "sub": context.get("user_id", ""),
+        "sub": str(context.get("user_id", "")),
         "iss": issuer,
         "iat": iat,
         "exp": exp,

fence/jwt/validate.py

@@ -1,3 +1,5 @@
+import logging
+
 import authutils.errors
 import authutils.token.keys
 import authutils.token.validate
@@ -109,6 +111,8 @@ def validate_jwt(
         ).get("iss")
     except jwt.InvalidTokenError as e:
         raise JWTError(e)
+    logging.error("token info: ")
+    logging.error(jwt.get_unverified_header(encoded_token))
     attempt_refresh = attempt_refresh and (token_iss != iss)
     public_key = public_key or authutils.token.keys.get_public_key_for_token(
         encoded_token, attempt_refresh=attempt_refresh, pkey_cache=pkey_cache

migrations/env.py

@@ -27,8 +27,9 @@
 
 target_metadata = Base.metadata
 
+test_config_path = os.environ.get("TEST_CONFIG_PATH")
 fence_config.load(
-    config_path=os.environ.get("TEST_CONFIG_PATH"),  # for tests
+    config_path=test_config_path,  # for tests
     search_folders=CONFIG_SEARCH_FOLDERS,  # for deployments
 )
 config.set_main_option("sqlalchemy.url", str(fence_config["DB"]))

pyproject.toml

@@ -57,7 +57,7 @@ boto = "*"
 # for testing with updated libaries as git repos:
 # foobar = {git = "https://github.com/uc-cdis/some-repo", rev = "feat/test"}
 
-[tool.poetry.dev-dependencies]
+[tool.poetry.group.dev.dependencies]
 addict = "^2.2.1"
 cdisutilstest = {git = "https://github.com/uc-cdis/cdisutils-test", tag = "2.0.0"}
 codacy-coverage = "^1.3.11"

tests/ci_commands_script.sh

@@ -2,4 +2,5 @@
 
 mkdir -p /var/tmp/uwsgi_flask_metrics/ || true
 export PROMETHEUS_MULTIPROC_DIR="/var/tmp/uwsgi_flask_metrics/"
-poetry run pytest -vv --cov=fence --cov-report xml tests
+echo "running tests"
+poetry run pytest -vv --cov=fence --cov-report xml .

tests/conftest.py

@@ -9,6 +9,7 @@
 import os
 import copy
 import time
+from pathlib import Path
 import flask
 from datetime import datetime
 import mock
@@ -437,7 +438,13 @@ def app(kid, rsa_private_key, rsa_public_key):
     mocker = Mocker()
     mocker.mock_functions()
 
-    root_dir = os.path.dirname(os.path.realpath(__file__))
+    # root_dir = os.path.dirname(os.path.realpath(__file__))
+    current_path = Path(__file__).resolve()
+    root_dir = current_path.parent.parent
+    os.chdir(root_dir)
+    x = str(root_dir)
+    test_dir = os.path.join(x, "tests")
+    test_config = os.path.join(test_dir, "test-fence-config.yaml")
 
     # delete the record operation from the data blueprint, because right now it calls a
     # whole bunch of stuff on the arborist client to do some setup for the uploader role
@@ -449,12 +456,12 @@ def app(kid, rsa_private_key, rsa_public_key):
     app_init(
         fence.app,
         test_settings,
-        root_dir=root_dir,
-        config_path=os.path.join(root_dir, "test-fence-config.yaml"),
+        root_dir=test_dir,
+        config_path=test_config,
     )
 
     # migrate the database to the latest version
-    os.environ["TEST_CONFIG_PATH"] = os.path.join(root_dir, "test-fence-config.yaml")
+    os.environ["TEST_CONFIG_PATH"] = os.path.join(test_dir, "test-fence-config.yaml")
     alembic_main(["--raiseerr", "upgrade", "head"])
 
     # We want to set up the keys so that the test application can load keys

tests/link/test_link.py

@@ -139,7 +139,7 @@ def test_google_link_session(app, client, encoded_creds_jwt):
     )
 
     assert flask.session.get("google_link") is True
-    assert flask.session.get("user_id") == user_id
+    assert flask.session.get("user_id") == str(user_id)
     assert flask.session.get("google_proxy_group_id") == proxy_group_id
     assert flask.session.get("redirect") == redirect
 

tests/oidc/core/user_info/test_userinfo.py

@@ -2,7 +2,6 @@
 import json
 
 import pytest
-from gen3authz.client.arborist.errors import ArboristError
 
 from fence.models import UserGoogleAccount
 
@@ -77,6 +76,8 @@ def test_userinfo_arborist_authz(
         "/user",
         headers={"Authorization": "Bearer " + encoded_creds_jwt["jwt"]},
     ).json
+    print(f"client: {client}")
+    print("testtesttesttest")
 
     actual_authz = resp.get("authz", {})
     actual_resources = resp.get("resources", [])

tests/test_drs.py

@@ -1,3 +1,5 @@
+import logging
+
 import flask
 import httpx
 import hashlib
@@ -67,15 +69,17 @@ def test_get_presigned_url_with_access_id(
     primary_google_service_account,
     cloud_manager,
     google_signed_url,
+        app,
 ):
     access_id = indexd_client["indexed_file_location"]
     test_guid = "1"
+    context_claims = utils.authorized_download_context_claims(
+                user_client.username, user_client.user_id
+            )
     user = {
         "Authorization": "Bearer "
         + jwt.encode(
-            utils.authorized_download_context_claims(
-                user_client.username, user_client.user_id
-            ),
+            context_claims,
             key=rsa_private_key,
             headers={"kid": kid},
             algorithm="RS256",
@@ -86,6 +90,12 @@ def test_get_presigned_url_with_access_id(
         "/ga4gh/drs/v1/objects/" + test_guid + "/access/" + access_id,
         headers=user,
     )
+    if res.status_code != 200:
+        logging.warning("Failed to get presigned url with access id")
+        log_info = res.status_code | {"kid": kid, "cc": context_claims}
+        logging.error(log_info)
+        logging.error("keys: ")
+        logging.error(str(list(list(app.jwt_public_keys.items())[0][1].items())))
     assert res.status_code == 200
 
 

tests/utils/__init__.py

@@ -287,7 +287,7 @@ def unauthorized_context_claims(user_name, user_id):
     iat, exp = iat_and_exp()
     return {
         "aud": [iss],
-        "sub": user_id,
+        "sub": str(user_id),
         "pur": "access",
         "iss": iss,
         "iat": iat,
@@ -320,7 +320,7 @@ def authorized_download_context_claims(user_name, user_id):
     iat, exp = iat_and_exp()
     return {
         "aud": [iss],
-        "sub": user_id,
+        "sub": str(user_id),
         "iss": iss,
         "iat": iat,
         "exp": exp,
@@ -353,7 +353,7 @@ def authorized_service_account_management_claims(user_name, user_id, client_id):
     iat, exp = iat_and_exp()
     return {
         "aud": [iss],
-        "sub": user_id,
+        "sub": str(user_id),
         "iss": iss,
         "iat": iat,
         "exp": exp,
@@ -403,7 +403,7 @@ def authorized_download_credentials_context_claims(
     iat, exp = iat_and_exp()
     return {
         "aud": [iss],
-        "sub": user_id,
+        "sub": str(user_id),
         "iss": iss,
         "iat": iat,
         "exp": exp,
@@ -436,7 +436,7 @@ def authorized_upload_context_claims(user_name, user_id):
     iat, exp = iat_and_exp()
     return {
         "aud": [iss],
-        "sub": user_id,
+        "sub": str(user_id),
         "iss": iss,
         "pur": "access",
         "iat": iat,