move backoff settings as well as other functions out of utils

fence/__init__.py

@@ -18,14 +18,8 @@
 logger = get_logger(__name__, log_level="debug")
 
 # Load the configuration *before* importing modules that rely on it
-from fence.config import config
+from fence.config import config, get_SQLAlchemyDriver
 from fence.settings import CONFIG_SEARCH_FOLDERS
-
-config.load(
-    config_path=os.environ.get("FENCE_CONFIG_PATH"),
-    search_folders=CONFIG_SEARCH_FOLDERS,
-)
-
 from fence.auth import logout, build_redirect_url
 from fence.metrics import metrics
 from fence.blueprints.data.indexd import S3IndexedFileLocation
@@ -48,7 +42,6 @@
 from fence.resources.storage import StorageManager
 from fence.resources.user.user_session import UserSessionInterface
 from fence.error_handler import get_error_response
-from fence.utils import get_SQLAlchemyDriver
 import fence.blueprints.admin
 import fence.blueprints.data
 import fence.blueprints.login

fence/config.py

@@ -1,4 +1,5 @@
 import os
+import requests
 from yaml import safe_load as yaml_load
 import urllib.parse
 
@@ -7,6 +8,10 @@
 
 from cdislogging import get_logger
 
+from fence.errors import NotFound
+from fence.utils import log_backoff_retry, log_backoff_giveup, exception_do_not_retry, logger
+from fence.settings import CONFIG_SEARCH_FOLDERS
+
 logger = get_logger(__name__)
 
 DEFAULT_CFG_PATH = os.path.join(
@@ -169,3 +174,71 @@ def _validate_parent_child_studies(dbgap_configs):
 
 
 config = FenceConfig(DEFAULT_CFG_PATH)
+config.load(
+    config_path=os.environ.get("FENCE_CONFIG_PATH"),
+    search_folders=CONFIG_SEARCH_FOLDERS,
+)
+
+# Default settings to control usage of backoff library.
+DEFAULT_BACKOFF_SETTINGS = {
+    "on_backoff": log_backoff_retry,
+    "on_giveup": log_backoff_giveup,
+    "max_tries": config["DEFAULT_BACKOFF_SETTINGS_MAX_TRIES"],
+    "giveup": exception_do_not_retry,
+}
+
+
+def get_SQLAlchemyDriver(db_conn_url):
+    from userdatamodel.driver import SQLAlchemyDriver
+
+    # override userdatamodel's `setup_db` function which creates tables
+    # and runs database migrations, because Alembic handles that now.
+    # TODO move userdatamodel code to Fence and remove dependencies to it
+    SQLAlchemyDriver.setup_db = lambda _: None
+    return SQLAlchemyDriver(db_conn_url)
+
+
+def send_email(from_email, to_emails, subject, text, smtp_domain):
+    """
+    Send email to group of emails using mail gun api.
+
+    https://app.mailgun.com/
+
+    Args:
+        from_email(str): from email
+        to_emails(list): list of emails to receive the messages
+        text(str): the text message
+        smtp_domain(dict): smtp domain server
+
+            {
+                "smtp_hostname": "smtp.mailgun.org",
+                "default_login": "postmaster@mailgun.planx-pla.net",
+                "api_url": "https://api.mailgun.net/v3/mailgun.planx-pla.net",
+                "smtp_password": "password", # pragma: allowlist secret
+                "api_key": "api key" # pragma: allowlist secret
+            }
+
+    Returns:
+        Http response
+
+    Exceptions:
+        KeyError
+
+    """
+    if smtp_domain not in config["GUN_MAIL"] or not config["GUN_MAIL"].get(
+        smtp_domain
+    ).get("smtp_password"):
+        raise NotFound(
+            "SMTP Domain '{}' does not exist in configuration for GUN_MAIL or "
+            "smtp_password was not provided. "
+            "Cannot send email.".format(smtp_domain)
+        )
+
+    api_key = config["GUN_MAIL"][smtp_domain].get("api_key", "")
+    email_url = config["GUN_MAIL"][smtp_domain].get("api_url", "") + "/messages"
+
+    return requests.post(
+        email_url,
+        auth=("api", api_key),
+        data={"from": from_email, "to": to_emails, "subject": subject, "text": text},
+    )

fence/models.py

@@ -57,9 +57,10 @@
     UserToGroup,
 )
 
-from fence import logger
-from fence.config import config
+from fence import logger, get_SQLAlchemyDriver
+from fence.config import config, logger
 from fence.errors import UserError
+from fence.utils import generate_client_credentials
 
 
 def query_for_user(session, username):
@@ -825,3 +826,69 @@ def populate_iss_sub_pair_to_user_table(target, connection, **kw):
         else:
             transaction.commit()
             logger.info("Population was successful")
+
+
+def create_client(
+    DB,
+    username=None,
+    urls=[],
+    name="",
+    description="",
+    auto_approve=False,
+    is_admin=False,
+    grant_types=None,
+    confidential=True,
+    arborist=None,
+    policies=None,
+    allowed_scopes=None,
+    expires_in=None,
+):
+    client_id, client_secret, hashed_secret = generate_client_credentials(confidential)
+    if arborist is not None:
+        arborist.create_client(client_id, policies)
+    driver = get_SQLAlchemyDriver(DB)
+    auth_method = "client_secret_basic" if confidential else "none"
+
+    allowed_scopes = allowed_scopes or config["CLIENT_ALLOWED_SCOPES"]
+    if not set(allowed_scopes).issubset(set(config["CLIENT_ALLOWED_SCOPES"])):
+        raise ValueError(
+            "Each allowed scope must be one of: {}".format(
+                config["CLIENT_ALLOWED_SCOPES"]
+            )
+        )
+
+    if "openid" not in allowed_scopes:
+        allowed_scopes.append("openid")
+        logger.warning('Adding required "openid" scope to list of allowed scopes.')
+
+    with driver.session as s:
+        user = None
+        if username:
+            user = query_for_user(session=s, username=username)
+            if not user:
+                user = User(username=username, is_admin=is_admin)
+                s.add(user)
+
+        if s.query(Client).filter(Client.name == name).first():
+            if arborist is not None:
+                arborist.delete_client(client_id)
+            raise Exception("client {} already exists".format(name))
+
+        client = Client(
+            client_id=client_id,
+            client_secret=hashed_secret,
+            user=user,
+            redirect_uris=urls,
+            allowed_scopes=" ".join(allowed_scopes),
+            description=description,
+            name=name,
+            auto_approve=auto_approve,
+            grant_types=grant_types,
+            is_confidential=confidential,
+            token_endpoint_auth_method=auth_method,
+            expires_in=expires_in,
+        )
+        s.add(client)
+        s.commit()
+
+    return client_id, client_secret

fence/resources/audit/client.py

@@ -4,10 +4,9 @@
 import requests
 import traceback
 
-from fence.config import config
+from fence.config import config, DEFAULT_BACKOFF_SETTINGS
 from fence.errors import InternalError
 from fence.resources.audit.utils import is_audit_enabled
-from fence.utils import DEFAULT_BACKOFF_SETTINGS
 
 
 class AuditServiceClient:

fence/resources/google/access_utils.py

@@ -15,7 +15,7 @@
 import fence
 from cdislogging import get_logger
 
-from fence.config import config
+from fence.config import config, DEFAULT_BACKOFF_SETTINGS
 from fence.errors import NotFound, NotSupported
 from fence.models import (
     User,
@@ -31,7 +31,7 @@
     get_monitoring_service_account_email,
     is_google_managed_service_account,
 )
-from fence.utils import get_valid_expiration_from_request, DEFAULT_BACKOFF_SETTINGS
+from fence.utils import get_valid_expiration_from_request
 
 logger = get_logger(__name__)
 

fence/resources/google/utils.py

@@ -19,7 +19,7 @@
 from userdatamodel.user import GoogleProxyGroup, User, AccessPrivilege
 
 from fence.auth import current_token
-from fence.config import config
+from fence.config import config, DEFAULT_BACKOFF_SETTINGS
 from fence.errors import NotSupported, InternalError, UserError
 from fence.models import (
     GoogleServiceAccount,
@@ -34,12 +34,10 @@
 )
 from fence.resources.google import STORAGE_ACCESS_PROVIDER_NAME
 from fence.errors import NotSupported, NotFound
-from fence.utils import get_SQLAlchemyDriver
+from fence import get_SQLAlchemyDriver
 
 from cdislogging import get_logger
 
-from fence.utils import DEFAULT_BACKOFF_SETTINGS
-
 logger = get_logger(__name__)
 
 

fence/resources/openid/ras_oauth2.py

@@ -16,7 +16,7 @@
 from gen3authz.client.arborist.errors import ArboristError
 
 
-from fence.config import config
+from fence.config import config, DEFAULT_BACKOFF_SETTINGS
 from fence.models import (
     GA4GHVisaV1,
     IdentityProvider,
@@ -26,7 +26,6 @@
     create_user,
 )
 from fence.jwt.validate import validate_jwt
-from fence.utils import DEFAULT_BACKOFF_SETTINGS
 from fence.errors import InternalError
 from .idp_oauth2 import Oauth2ClientBase
 

fence/scripting/fence_create.py

@@ -51,17 +51,16 @@
     ServiceAccountToGoogleBucketAccessGroup,
     query_for_user,
     GA4GHVisaV1,
-    get_client_expires_at,
+    get_client_expires_at, create_client,
 )
 from fence.scripting.google_monitor import email_users_without_access, validation_check
 from fence.config import config
 from fence.sync.sync_users import UserSyncer
 from fence.utils import (
-    create_client,
     get_valid_expiration,
     generate_client_credentials,
-    get_SQLAlchemyDriver,
 )
+from fence import get_SQLAlchemyDriver
 from sqlalchemy.orm.attributes import flag_modified
 from gen3authz.client.arborist.client import ArboristClient
 

fence/scripting/google_monitor.py

@@ -13,6 +13,7 @@
 
 from cdislogging import get_logger
 
+import fence.config
 from fence.resources.google.validity import (
     GoogleProjectValidity,
     GoogleServiceAccountValidity,
@@ -514,7 +515,7 @@ def _send_emails_informing_service_account_removal(
                 for reason in removal_reasons:
                     content += "\n\t\t\t - {}".format(reason)
 
-    return utils.send_email(from_email, to_emails, subject, content, domain)
+    return fence.config.send_email(from_email, to_emails, subject, content, domain)
 
 
 def _get_users_without_access(db, auth_ids, user_emails, check_linking):
@@ -609,7 +610,7 @@ def email_user_without_access(user_email, projects, google_project_id):
     text = config["PROBLEM_USER_EMAIL_NOTIFICATION"]["content"]
     content = text.format(google_project_id, ",".join(projects))
 
-    return utils.send_email(from_email, to_emails, subject, content, domain)
+    return fence.config.send_email(from_email, to_emails, subject, content, domain)
 
 
 def email_users_without_access(

fence/sync/sync_users.py

@@ -26,7 +26,7 @@
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy import func
 
-from fence.config import config
+from fence.config import config, DEFAULT_BACKOFF_SETTINGS
 from fence.models import (
     AccessPrivilege,
     AuthorizationProvider,
@@ -44,7 +44,7 @@
 from fence.resources.google.access_utils import GoogleUpdateException
 from fence.sync import utils
 from fence.sync.passport_sync.ras_sync import RASVisa
-from fence.utils import get_SQLAlchemyDriver, DEFAULT_BACKOFF_SETTINGS
+from fence import get_SQLAlchemyDriver
 
 
 def _format_policy_id(path, privilege):