diff --git a/docs/google_architecture.md b/docs/google_architecture.md index 49c39997a..b00012ff0 100644 --- a/docs/google_architecture.md +++ b/docs/google_architecture.md @@ -17,7 +17,7 @@ We'll talk about each one of those in-depth here (and even delve into the intern ### Fence -> cirrus -> Google: A library wrapping Google's API -We have a library that wraps Google's public API called [cirrus](https://github.com/uc-cdis/cirrus). Our design is such that fence does not hit Google's API directly, but goes through cirrus. For all of cirrus's features to work, a very specific setup is required, which is detailed in cirrus's README. +We have a library that wraps Google's public API called [cirrus](https://github.com/uc-cdis/cirrus). Our design is such that fence does not hit Google's API directly, but goes through gen3cirrus. For all of cirrus's features to work, a very specific setup is required, which is detailed in cirrus's README. Essentially, cirrus requires a Google Cloud Identity account (for group management) and Google Cloud Platform project(s). In order to automate group management in Google Cloud Identity with cirrus, you must go through a manual process of allowing API access and delegating a specific service account from a Google Cloud Platform project to have group management authority. Details can be found in cirrus's README. diff --git a/fence/blueprints/data/indexd.py b/fence/blueprints/data/indexd.py index 93cf60525..cb4bbfeeb 100755 --- a/fence/blueprints/data/indexd.py +++ b/fence/blueprints/data/indexd.py @@ -6,8 +6,8 @@ from sqlalchemy.sql.functions import user from cached_property import cached_property -import cirrus -from cirrus import GoogleCloudManager +import gen3cirrus +from gen3cirrus import GoogleCloudManager from cdislogging import get_logger from cdispyutils.config import get_value from cdispyutils.hmac4 import generate_aws_presigned_url @@ -162,7 +162,7 @@ def get_signed_url_for_file( _log_signed_url_data_info( indexed_file=indexed_file, user_sub=flask.g.audit_data.get("sub", ""), - requested_protocol=requested_protocol + requested_protocol=requested_protocol, ) return {"url": signed_url} @@ -1197,7 +1197,7 @@ def _generate_anonymous_google_storage_signed_url( ): # we will use the main fence SA service account to sign anonymous requests private_key = get_google_app_creds() - final_url = cirrus.google_cloud.utils.get_signed_url( + final_url = gen3cirrus.google_cloud.utils.get_signed_url( resource_path, http_verb, expires_in, @@ -1338,7 +1338,7 @@ def _generate_google_storage_signed_url( if config["BILLING_PROJECT_FOR_SIGNED_URLS"] and not r_pays_project: r_pays_project = config["BILLING_PROJECT_FOR_SIGNED_URLS"] - final_url = cirrus.google_cloud.utils.get_signed_url( + final_url = gen3cirrus.google_cloud.utils.get_signed_url( resource_path, http_verb, expires_in, diff --git a/fence/blueprints/google.py b/fence/blueprints/google.py index a3c8b2348..3e3def67d 100644 --- a/fence/blueprints/google.py +++ b/fence/blueprints/google.py @@ -7,9 +7,9 @@ import flask from flask_restful import Resource -from cirrus import GoogleCloudManager -from cirrus.errors import CirrusNotFound -from cirrus.google_cloud.errors import GoogleAPIError +from gen3cirrus import GoogleCloudManager +from gen3cirrus.errors import CirrusNotFound +from gen3cirrus.google_cloud.errors import GoogleAPIError from fence.auth import current_token, require_auth_header from fence.restful import RestfulApi diff --git a/fence/blueprints/link.py b/fence/blueprints/link.py index 99ac4c9fa..a9854f067 100644 --- a/fence/blueprints/link.py +++ b/fence/blueprints/link.py @@ -6,7 +6,7 @@ from cdislogging import get_logger -from cirrus import GoogleCloudManager +from gen3cirrus import GoogleCloudManager from fence.blueprints.login.redirect import validate_redirect from fence.restful import RestfulApi from fence.errors import NotFound diff --git a/fence/blueprints/storage_creds/google.py b/fence/blueprints/storage_creds/google.py index b9c67fee3..49c4cadd4 100644 --- a/fence/blueprints/storage_creds/google.py +++ b/fence/blueprints/storage_creds/google.py @@ -6,8 +6,8 @@ from flask_restful import Resource from flask import current_app -from cirrus import GoogleCloudManager -from cirrus.config import config as cirrus_config +from gen3cirrus import GoogleCloudManager +from gen3cirrus.config import config as cirrus_config from fence.config import config from fence.auth import require_auth_header diff --git a/fence/config.py b/fence/config.py index b4deeeb22..21bcf42cb 100644 --- a/fence/config.py +++ b/fence/config.py @@ -2,7 +2,7 @@ from yaml import safe_load as yaml_load import urllib.parse -import cirrus +import gen3cirrus from gen3config import Config from cdislogging import get_logger @@ -92,7 +92,7 @@ def post_process(self): if self._configs.get("MOCK_STORAGE", False): self._configs["STORAGE_CREDENTIALS"] = {} - cirrus.config.config.update(**self._configs.get("CIRRUS_CFG", {})) + gen3cirrus.config.config.update(**self._configs.get("CIRRUS_CFG", {})) # if we have a default google project for billing requester pays, we should # NOT allow end-users to have permission to create Temporary Google Service diff --git a/fence/resources/admin/admin_users.py b/fence/resources/admin/admin_users.py index f86f45178..373912c17 100644 --- a/fence/resources/admin/admin_users.py +++ b/fence/resources/admin/admin_users.py @@ -1,6 +1,6 @@ from cdislogging import get_logger -from cirrus import GoogleCloudManager -from cirrus.google_cloud.utils import get_proxy_group_name_for_user +from gen3cirrus import GoogleCloudManager +from gen3cirrus.google_cloud.utils import get_proxy_group_name_for_user from fence.config import config from fence.errors import NotFound, UserError, UnavailableError from fence.models import ( @@ -363,7 +363,7 @@ def delete_user(current_session, username): # and check if it exists in cirrus, in case Fence db just # didn't know about it. logger.debug( - "Could not find Google proxy group for this user in Fence db. Checking cirrus..." + "Could not find Google proxy group for this user in Fence db. Checking gen3cirrus..." ) pgname = get_proxy_group_name_for_user( user.id, user.username, prefix=config["GOOGLE_GROUP_PREFIX"] @@ -377,7 +377,7 @@ def delete_user(current_session, username): if not gpg_email: logger.info( - "Could not find Google proxy group for user in Fence db or in cirrus. " + "Could not find Google proxy group for user in Fence db or in gen3cirrus. " "Assuming Google not in use as IdP. Proceeding with Fence deletes." ) else: diff --git a/fence/resources/google/access_utils.py b/fence/resources/google/access_utils.py index be1c77978..212688845 100644 --- a/fence/resources/google/access_utils.py +++ b/fence/resources/google/access_utils.py @@ -8,10 +8,10 @@ from urllib.parse import unquote import traceback -from cirrus.google_cloud.iam import GooglePolicyMember -from cirrus.google_cloud.errors import GoogleAPIError -from cirrus.google_cloud.iam import GooglePolicy -from cirrus import GoogleCloudManager +from gen3cirrus.google_cloud.iam import GooglePolicyMember +from gen3cirrus.google_cloud.errors import GoogleAPIError +from gen3cirrus.google_cloud.iam import GooglePolicy +from gen3cirrus import GoogleCloudManager import fence from cdislogging import get_logger @@ -218,7 +218,7 @@ def get_google_project_valid_users_and_service_accounts( Will make call to Google API if membership is None Return: - List[cirrus.google_cloud.iam.GooglePolicyMember]: Members on the + List[gen3cirrus.google_cloud.iam.GooglePolicyMember]: Members on the google project Raises: diff --git a/fence/resources/google/utils.py b/fence/resources/google/utils.py index 74e43cb3c..ae1be0a94 100644 --- a/fence/resources/google/utils.py +++ b/fence/resources/google/utils.py @@ -9,9 +9,9 @@ from sqlalchemy import desc, func from cdislogging import get_logger -from cirrus import GoogleCloudManager -from cirrus.google_cloud.iam import GooglePolicyMember -from cirrus.google_cloud.utils import ( +from gen3cirrus import GoogleCloudManager +from gen3cirrus.google_cloud.iam import GooglePolicyMember +from gen3cirrus.google_cloud.utils import ( get_valid_service_account_id_for_client, get_valid_service_account_id_for_user, ) diff --git a/fence/resources/google/validity.py b/fence/resources/google/validity.py index 0cf372032..505b83b2e 100644 --- a/fence/resources/google/validity.py +++ b/fence/resources/google/validity.py @@ -27,7 +27,7 @@ is_user_member_of_google_project, is_user_member_of_all_google_projects, ) -from cirrus.google_cloud import GoogleCloudManager +from gen3cirrus.google_cloud import GoogleCloudManager from cdislogging import get_logger diff --git a/fence/resources/storage/__init__.py b/fence/resources/storage/__init__.py index acb402739..4aeaad268 100644 --- a/fence/resources/storage/__init__.py +++ b/fence/resources/storage/__init__.py @@ -1,7 +1,7 @@ import copy from functools import wraps -from storageclient import get_client +from fence.resources.storage.storageclient import get_client from fence.models import ( CloudProvider, diff --git a/fence/resources/storage/storageclient/__init__.py b/fence/resources/storage/storageclient/__init__.py new file mode 100644 index 000000000..8418f8cf5 --- /dev/null +++ b/fence/resources/storage/storageclient/__init__.py @@ -0,0 +1,12 @@ +from fence.resources.storage.storageclient.cleversafe import CleversafeClient +from fence.resources.storage.storageclient.google import GoogleCloudStorageClient + + +def get_client(config=None, backend=None): + try: + clients = {"cleversafe": CleversafeClient, "google": GoogleCloudStorageClient} + return clients[backend](config) + except KeyError as ex: + raise NotImplementedError( + "The input storage is currently not supported!: {0}".format(ex) + ) diff --git a/fence/resources/storage/storageclient/base.py b/fence/resources/storage/storageclient/base.py new file mode 100644 index 000000000..d327be597 --- /dev/null +++ b/fence/resources/storage/storageclient/base.py @@ -0,0 +1,224 @@ +from abc import abstractmethod, abstractproperty, ABCMeta +from .errors import ClientSideError +import logging +from cdislogging import get_logger + + +def handle_request(fun): + """ + Exception treatment for the REST API calls + """ + + def wrapper(self, *args, **kwargs): + """ + We raise an exception when + the code on the client side fails + Server side errors are taken care of + through response codes + """ + try: + return fun(self, *args, **kwargs) + except Exception as req_exception: + self.logger.exception("internal error") + raise ClientSideError(str(req_exception)) + + return wrapper + + +class StorageClient(object, metaclass=ABCMeta): + """Abstract storage client class""" + + def __init__(self, cls_name): + self.logger = get_logger(cls_name) + self.logger.setLevel(logging.DEBUG) + + @abstractproperty + def provider(self): + """ + Name of the storage provider. eg: ceph + """ + msg = "Provider not implemented" + raise NotImplementedError(msg) + + @abstractmethod + def get_user(self, username): + """ + Get a user + :returns: a User object if the user exists, else None + """ + msg = "get_user not implemented" + raise NotImplementedError(msg) + + @abstractmethod + def delete_user(self, username): + """ + Delete a user + :returns: None + :raise: + :NotFound: the user is not found + """ + msg = "delete_user not implemented" + raise NotImplementedError(msg) + + @abstractmethod + def create_user(self, username): + """ + Create a user + :returns: User object + """ + msg = "create_user not implemented" + raise NotImplementedError(msg) + + @abstractmethod + def list_users(self): + """ + List users + :returns: a list of User objects + """ + msg = "list_users not implemented" + raise NotImplementedError(msg) + + @abstractmethod + def get_or_create_user(self, username): + """ + Tries to retrieve a user. + If the user is not found, a new one + is created and returned + """ + msg = "get_or_create_user not implemented" + raise NotImplementedError(msg) + + @abstractmethod + def create_keypair(self, username): + """ + Creates a keypair for the user, and + returns it + """ + msg = "create_keypair not implemented" + raise NotImplementedError(msg) + + @abstractmethod + def delete_keypair(self, username, access_key): + """ + Deletes a keypair from the user and + doesn't return anything + """ + msg = "delete_keypair not implemented" + raise NotImplementedError(msg) + + @abstractmethod + def add_bucket_acl(self, bucket, username, access=None): + """ + Tries to grant a user access to a bucket + """ + msg = "add_bucket_acl not implemented" + raise NotImplementedError(msg) + + @abstractmethod + def has_bucket_access(self, bucket, user_id): + """ + Check if the user appears in the acl + : returns Bool + """ + msg = "has_bucket_access not implemented" + raise NotImplementedError(msg) + + @abstractmethod + def list_buckets(self): + """ + Return a list of Bucket objects + : [bucket1, bucket2,...] + """ + msg = "list_buckets not implemented" + raise NotImplementedError(msg) + + @abstractmethod + def delete_all_keypairs(self, user): + """ + Remove all the keys from a user + : returns None + """ + msg = "delete_all_keypairs not implemented" + raise NotImplementedError(msg) + + @abstractmethod + def get_bucket(self, bucket): + """ + Return a bucket from the storage + """ + msg = "get_bucket not implemented" + raise NotImplementedError(msg) + + @abstractmethod + def get_or_create_bucket(self, bucket, access_key=None, secret_key=None): + """ + Tries to retrieve a bucket and if fit fails, + creates and returns one + """ + msg = "get_or_create_bucket not implemented" + raise NotImplementedError(msg) + + @abstractmethod + def get_bucket(self, bucket, access_key=None, secret_key=None): + """ + Tries to retrieve a bucket and if fit fails, + creates and returns one + """ + msg = "get_bucket not implemented" + raise NotImplementedError(msg) + + @abstractmethod + def edit_bucket_template(self, template_id, **kwargs): + """ + Change the parameters for the template used to create + the buckets + """ + msg = "edit_bucket_template not implemented" + raise NotImplementedError(msg) + + @abstractmethod + def update_bucket_acl(self, bucket, user_list): + """ + Add acl's for the list of users + """ + msg = "update_bucket_acl not implemented" + raise NotImplementedError(msg) + + @abstractmethod + def set_bucket_quota(self, bucket, quota_unit, quota): + """ + Set quota for the entire bucket + """ + msg = "set_bucket_quota not implemented" + raise NotImplementedError(msg) + + @abstractmethod + def delete_bucket_acl(self, bucket, user): + """ + Set quota for the entire bucket + """ + msg = "delete_bucket_acl not implemented" + raise NotImplementedError(msg) + + +class User(object): + def __init__(self, username): + """ + - permissions {'bucketname': 'PERMISSION'} + - keys [{'access_key': abc,'secret_key': 'def'}] + """ + self.username = username + self.permissions = {} + self.keys = [] + self.id = None + + +class Bucket(object): + def __init__(self, name, bucket_id, quota): + """ + Simple bucket representation + Quota is in TiBs or such units + """ + self.name = name + self.id = bucket_id + self.quota = quota diff --git a/fence/resources/storage/storageclient/cleversafe.py b/fence/resources/storage/storageclient/cleversafe.py new file mode 100644 index 000000000..104d1d94d --- /dev/null +++ b/fence/resources/storage/storageclient/cleversafe.py @@ -0,0 +1,525 @@ +""" +Connection manager for the Cleversafe storage system +Since it is compatible with S3, we will be using boto. +""" +from boto import connect_s3 +from boto.s3 import connection +from boto.exception import S3ResponseError +from boto.s3.acl import Grant +import requests +from urllib.parse import urlencode +import json +from .base import StorageClient, User, Bucket, handle_request +from .errors import RequestError, NotFoundError + + +class CleversafeClient(StorageClient): + """ + Connection manager for Cleversafe. + Isolates differences from other connectors + """ + + def __init__(self, config): + """ + Creation of the manager. Since it is only s3 compatible + we need to specify the endpoint in the config + """ + super(CleversafeClient, self).__init__(__name__) + self._config = config + self._host = config["host"] + self._public_host = config["public_host"] + self._access_key = config["aws_access_key_id"] + self._secret_key = config["aws_secret_access_key"] + self._username = config["username"] + self._password = config["password"] + self._permissions_order = { + "read-storage": 1, + "write-storage": 2, + "admin-storage": 3, + "disabled": 0, + } + self._permissions_value = ["disabled", "readOnly", "readWrite", "owner"] + self._auth = requests.auth.HTTPBasicAuth(self._username, self._password) + self._conn = connect_s3( + aws_access_key_id=self._access_key, + aws_secret_access_key=self._secret_key, + host=self._public_host, + calling_format=connection.OrdinaryCallingFormat(), + ) + self._bucket_name_id_table = {} + self._update_bucket_name_id_table() + self._user_name_id_table = {} + self._user_id_name_table = {} + self._update_user_name_id_table() + + def _update_user_name_id_table(self): + """ + Update the name-id translation table for users + """ + response = self._request("GET", "listAccounts.adm") + if response.status_code == 200: + jsn = json.loads(response.text) + self._user_name_id_table = {} + for user in jsn["responseData"]["accounts"]: + self._user_name_id_table[user["name"]] = user["id"] + self._user_id_name_table[user["id"]] = user["name"] + self.logger.debug(self._user_name_id_table) + self.logger.debug(self._user_id_name_table) + else: + msg = "List users failed on update cache with code {0}" + self.logger.error(msg.format(response.status_code)) + raise RequestError(response.text, response.status_code) + + def _update_bucket_name_id_table(self): + """ + Update the name-id translation table for buckets + """ + response = self._request("GET", "listVaults.adm") + if response.status_code == 200: + jsn = json.loads(response.text) + self._bucket_name_id_table = {} + for user in jsn["responseData"]["vaults"]: + self._bucket_name_id_table[user["name"]] = user["id"] + self.logger.debug(self._bucket_name_id_table) + else: + msg = "List vaults failed on update cache with code {0}" + self.logger.error(msg.format(response.status_code)) + raise RequestError(response.text, response.status_code) + + def _get_bucket_id(self, name): + """ + Tries to return the id from the table + If the cache misses, it updates it and + tries again + TODO OPTIMIZATION get the user information + from the update itself + """ + try: + return self._bucket_name_id_table[name] + except KeyError: + self._update_bucket_name_id_table() + return self._bucket_name_id_table[name] + + def _get_user_id(self, name): + """ + Tries to return the id from the table + If the cache misses, it updates it and + tries again + """ + try: + return self._user_name_id_table[name] + except KeyError: + self._update_user_name_id_table() + return self._user_name_id_table[name] + + def _get_user_by_id(self, uid): + """ + Fetches the user by id from the REST API + """ + response = self._request("GET", "viewSystem.adm", itemType="account", id=uid) + if response.status_code == 200: + user = json.loads(response.text) + try: + return self._populate_user(user["responseData"]["accounts"][0]) + except: + # Request OK but User not found + return None + else: + self.logger.error( + "get_user failed with code: {code}".format(code=response.status_code) + ) + raise RequestError(response.text, response.status_code) + + def _populate_user(self, data): + """ + Populates a new user with the data provided + in a jsonreponse + """ + try: + new_user = User(data["name"]) + new_user.id = data["id"] + for key in data["accessKeys"]: + new_key = { + "access_key": key["accessKeyId"], + "secret_key": key["secretAccessKey"], + } + new_user.keys.append(new_key) + vault_roles = [] + for role in data["roles"]: + if role["role"] == "vaultUser": + vault_roles = role["vaultPermissions"] + for vault_permission in vault_roles: + vault_response = self._get_bucket_by_id(vault_permission["vault"]) + vault = json.loads(vault_response.text) + new_user.permissions[ + vault["responseData"]["vaults"][0]["name"] + ] = vault_permission["permission"] + return new_user + except KeyError as key_e: + msg = "Failed to parse the user data. Check user fields inside the accounts section" + self.logger.error(msg) + raise RequestError(str(key_e), "200") + + def _get_bucket_by_id(self, vid): + """ + Get bucket by id + """ + response = self._request("GET", "viewSystem.adm", itemType="vault", id=vid) + if response.status_code == 200: + return response + else: + msg = "Get bucket by id failed with code: {0}" + self.logger.error(msg.format(response.status_code)) + raise RequestError(response.text, response.status_code) + + @handle_request + def _request(self, method, operation, payload=None, **kwargs): + """ + Compose the request and send it + """ + base_url = "https://{host}/manager/api/json/1.0/{oper}".format( + host=self._host, oper=operation + ) + url = base_url + "?" + urlencode(dict(**kwargs)) + return requests.request( + method, url, auth=self._auth, data=payload, verify=False + ) # self-signed certificate + + @property + def provider(self): + """ + Returns the type of storage + """ + return "Cleversafe" + + def list_users(self): + """ + Returns a list with all the users, in User objects + """ + response = self._request("GET", "listAccounts.adm") + if response.status_code == 200: + jsn = json.loads(response.text) + user_list = [] + for user in jsn["responseData"]["accounts"]: + new_user = self._populate_user(user) + user_list.append(new_user) + return user_list + else: + msg = "List buckets failed with code {0}" + self.logger.error(msg.format(response.status_code)) + raise RequestError(response.text, response.status_code) + + def has_bucket_access(self, bucket, username): + """ + Find if a user is in the grants list of the acl for + a certain bucket. + Please keep in mind that buckets must be all lowercase + """ + vault_id = self._get_bucket_id(bucket) + vault = json.loads(self._get_bucket_by_id(vault_id).text) + user_id = self._get_user_id(username) + for permission in vault["responseData"]["vaults"][0]["accessPermissions"]: + if permission["principal"]["id"] == user_id: + return True + return False + + def get_user(self, name): + """ + Gets the information from the user including + but not limited to: + - username + - name + - roles + - permissions + - access_keys + - emailxs + """ + try: + uid = self._get_user_id(name) + except KeyError: + return None + return self._get_user_by_id(uid) + + def list_buckets(self): + """ + Lists all the vaults(buckets) and their information + """ + response = self._request("GET", "listVaults.adm") + if response.status_code == 200: + buckets = json.loads(response.text) + bucket_list = [] + for buck in buckets["responseData"]["vaults"]: + new_bucket = Bucket(buck["name"], buck["id"], buck["hardQuota"]) + bucket_list.append(new_bucket) + return bucket_list + else: + self.logger.error( + "List buckets failed with code: {code}".format( + code=response.status_code + ) + ) + raise RequestError(response.text, response.status_code) + + def create_user(self, name): + """ + Creates a user + TODO Input sanitazion for parameters + """ + data = {"name": name, "usingPassword": "false"} + response = self._request("POST", "createAccount.adm", payload=data) + if response.status_code == 200: + parsed_reply = json.loads(response.text) + user_id = parsed_reply["responseData"]["id"] + self._update_user_name_id_table() + return self._get_user_by_id(user_id) + else: + self.logger.error( + "User creation failed with code: {0}".format(response.status_code) + ) + raise RequestError(response.text, response.status_code) + + def delete_user(self, name): + """ + Eliminate a user account + Requires the password from the account requesting the deletion + """ + uid = self._get_user_id(name) + data = {"id": uid, "password": self._config["password"]} + response = self._request("POST", "deleteAccount.adm", payload=data) + if response.status_code == 200: + self._update_user_name_id_table() + return None + else: + self.logger.error( + "Delete user failed with code: {0}".format(response.status_code) + ) + raise RequestError(response.text, response.status_code) + + def delete_keypair(self, name, access_key): + """ + Remove the give key/secret that match the key id + """ + uid = self._get_user_id(name) + data = {"id": uid, "accessKeyId": access_key, "action": "remove"} + response = self._request("POST", "editAccountAccessKey.adm", payload=data) + if response.status_code == 200: + return None + else: + self.logger.error( + "Delete keypair failed with code: {0}".format(response.status_code) + ) + raise RequestError(response.text, response.status_code) + + def delete_all_keypairs(self, name): + """ + Remove all keys from a give user + TODO Make this robust against possible errors so most of the keys are deleted + or retried + """ + user = self.get_user(name) + exception = False + responses_list = [] + responses_codes = [] + for key in user.keys: + try: + self.delete_keypair(user.username, key["access_key"]) + except RequestError as exce: + exception = True + msg = "Remove all keys failed for one key" + self.logger.error(msg.format(exce.code)) + responses_list.append(str(exce)) + responses_codes.append(exce.code) + if exception: + raise RequestError(responses_list, responses_codes) + else: + return None + + def create_keypair(self, name): + """ + Add a new key/secret pair + """ + uid = self._get_user_id(name) + data = {"id": uid, "action": "add"} + response = self._request("POST", "editAccountAccessKey.adm", payload=data) + if response.status_code == 200: + jsn = json.loads(response.text) + keypair = { + "access_key": jsn["responseData"]["accessKeyId"], + "secret_key": jsn["responseData"]["secretAccessKey"], + } + return keypair + else: + msg = "Create keypair failed with error code: {0}" + self.logger.error(msg.format(response.status_code)) + raise RequestError(response.text, response.status_code) + + def get_bucket(self, bucket): + """ + Retrieves the information from the bucket matching the name + """ + try: + bucket_id = self._get_bucket_id(bucket) + """at this point we have all we need for the initial + Bucket object, but for coherence, we keep this last call. + Feel free to get more information from response.text""" + response = self._get_bucket_by_id(bucket_id) + vault = json.loads(response.text) + return Bucket( + bucket, bucket_id, vault["responseData"]["vaults"][0]["hardQuota"] + ) + except KeyError as exce: + self.logger.error("Get bucket not found on cache") + raise RequestError(str(exce), "NA") + except RequestError as exce: + self.logger.error("Get bucket failed retrieving bucket info") + raise exce + + def get_or_create_user(self, name): + """ + Tries to get a user and if it doesn't exist, creates a new one + """ + user = self.get_user(name) + if user != None: + return user + else: + return self.create_user(name) + + def get_or_create_bucket(self, bucket_name, access_key=None, secret_key=None): + """ + Tries to retrieve a bucket and if it doesn't exist, creates a new one + """ + bucket = self.get_bucket(bucket_name) + if bucket != None: + return bucket + else: + if not access_key: + access_key = self._access_key + if not secret_key: + secret_key = self._secret_key + return self.create_bucket(bucket_name, access_key, secret_key) + + def create_bucket(self, bucket_name, access_key=None, secret_key=None): + """ + Requires a default template created on cleversafe + """ + if not access_key: + access_key = self._access_key + if not secret_key: + secret_key = self._secret_key + creds = {"host": self._public_host} + creds["aws_access_key_id"] = access_key + creds["aws_secret_access_key"] = secret_key + conn = connect_s3(calling_format=connection.OrdinaryCallingFormat(), **creds) + try: + bucket = conn.create_bucket(bucket_name) + self._update_bucket_name_id_table() + return bucket + except S3ResponseError as exce: + msg = "Create bucket failed with error code: {0}" + self.logger.error(msg.format(exce.error_code)) + raise RequestError(str(exce), exce.error_code) + + def edit_bucket_template(self, default_template_id, **kwargs): + """ + Change the desired parameters of the default template + This will affect every new bucket creation + The idea is to have only one template, the default one, and + modify it accordingly + """ + data = kwargs + data["id"] = default_template_id + response = self._request("POST", "editVaultTemplate.adm", payload=data) + if response.status_code == 200: + return response + else: + msg = "Edit bucket template failed with code: {0}" + self.logger.error(msg.format(response.status_code)) + raise RequestError(response.text, response.status_code) + + def update_bucket_acl(self, bucket, new_grants): + """ + Get an acl object and add the missing credentials + to the one retrieved from the target bucket + new_grants contains a list of users and permissions + [('user1', ['read-storage', 'write-storage']),...] + """ + user_id_list = [] + for user in new_grants: + user_id_list.append(self._get_user_id(user[0])) + bucket_id = self._get_bucket_id(bucket) + response = self._get_bucket_by_id(bucket_id) + vault = json.loads(response.text)["responseData"]["vaults"][0] + disable = [] + for permission in vault["accessPermissions"]: + uid = permission["principal"]["id"] + permit_type = permission["permission"] + if uid not in user_id_list or permit_type != "owner": + disable.append((self._user_id_name_table[uid], ["disabled"])) + for user in disable: + self.add_bucket_acl(bucket, user[0], user[1]) + for user in new_grants: + self.add_bucket_acl(bucket, user[0], user[1]) + + def set_bucket_quota(self, bucket, quota_unit, quota): + """ + Set qouta for the entire bucket/vault + """ + bid = self._get_bucket_id(bucket) + data = {"hardQuotaSize": quota, "hardQuotaUnit": quota_unit, "id": bid} + response = self._request("POST", "editVault.adm", payload=data) + if response.status_code == 200: + return response + else: + msg = "Set bucket quota failed with code: {0}" + self.logger.error(msg.format(response.status_code)) + raise RequestError(response.text, response.status_code) + + def add_bucket_acl(self, bucket, username, access=[]): + """ + Add permissions to a user on the bucket ACL + """ + try: + bucket_param = "vaultUserPermissions[{0}]".format( + self._get_bucket_id(bucket) + ) + except KeyError: + msg = "Bucket {0} wasn't found on the database" + self.logger.error(msg.format(bucket)) + raise NotFoundError(msg.format(bucket)) + try: + access_lvl = max(self._permissions_order[role] for role in access) + data = { + "id": self._get_user_id(username), + bucket_param: self._permissions_value[access_lvl], + } + if access_lvl == 3: + data["rolesMap[vaultProvisioner]"] = "true" + except KeyError: + msg = "User {0} wasn't found on the database" + self.logger.error(msg.format(username)) + raise NotFoundError(msg.format(username)) + response = self._request("POST", "editAccount.adm", payload=data) + if response.status_code != 200: + msg = "Error trying to change buket permissions for user {0}" + self.logger.error(msg.format(username)) + raise RequestError(msg.format(username), response.status_code) + + def delete_bucket(self, bucket_name): + """ + Delete a bucket + """ + bucket_id = self._get_bucket_id(bucket_name) + data = {"id": bucket_id, "password": self._password} + response = self._request("POST", "deleteVault.adm", payload=data) + self._update_bucket_name_id_table() + if response.status_code != 200: + msg = "Error trying to delete vault {bucket}" + self.logger.error(msg.format(bucket_name)) + raise RequestError(msg.format(bucket_name), response.status_code) + + def delete_bucket_acl(self, bucket, username): + """ + Remove permission from a bucket + """ + self.add_bucket_acl(bucket, username, ["disabled"]) + return None diff --git a/fence/resources/storage/storageclient/errors.py b/fence/resources/storage/storageclient/errors.py new file mode 100644 index 000000000..8f9eddb78 --- /dev/null +++ b/fence/resources/storage/storageclient/errors.py @@ -0,0 +1,14 @@ +class RequestError(Exception): + def __init__(self, message, code): + self.message = message + self.code = code + + +class NotFoundError(RequestError): + def __init__(self, message): + super().__init__(message, 404) + + +class ClientSideError(RequestError): + def __init__(self, message): + super().__init__(message, 400) diff --git a/fence/resources/storage/storageclient/google.py b/fence/resources/storage/storageclient/google.py new file mode 100644 index 000000000..0733f9a09 --- /dev/null +++ b/fence/resources/storage/storageclient/google.py @@ -0,0 +1,217 @@ +from fence.resources.storage.storageclient.base import StorageClient +from fence.resources.storage.storageclient.errors import RequestError +from gen3cirrus import GoogleCloudManager + + +class UserProxy(object): + def __init__(self, username): + self.username = username + + +class GoogleCloudStorageClient(StorageClient): + def __init__(self, config): + super(GoogleCloudStorageClient, self).__init__(__name__) + self._config = config + self.google_project_id = config.get("google_project_id") + + @property + def provider(self): + """ + Returns the type of storage + """ + return "GoogleCloudStorage" + + def get_user(self, username): + """ + Get a user + + Args: + username (str): An email address representing a User's Google + Proxy Group (e.g. a single Google Group to hold a single + user's diff identities). + + Returns: + UserProxy: a UserProxy object if the user exists, else None + """ + user_proxy = None + + with GoogleCloudManager(project_id=self.google_project_id) as g_mgr: + user_proxy_response = g_mgr.get_group(username) + if user_proxy_response.get("email"): + user_proxy = UserProxy(username=user_proxy_response.get("email")) + + return user_proxy + + def delete_user(self, username): + """ + Delete a user + :returns: None + :raise: + :NotFound: the user is not found + """ + msg = "delete_user not implemented" + raise NotImplementedError(msg) + + def create_user(self, username): + """ + Create a user + :returns: User object + """ + msg = "create_user not implemented" + raise NotImplementedError(msg) + + def list_users(self): + """ + List users + :returns: a list of User objects + """ + msg = "list_users not implemented" + raise NotImplementedError(msg) + + def get_or_create_user(self, username): + """ + Tries to retrieve a user. + + WARNING: If the user is not found, this DOES NOT CREATE ONE. + + Google architecture requires that a separate process populate + a proxy Google group per user. If it doesn't exist, we can't create it + here. + """ + user_proxy = self.get_user(username) + if not user_proxy: + raise Exception( + "Unable to determine User's Google Proxy group. Cannot create " + "here. Another process should create proxy groups for " + "new users. Username provided: {}".format(username) + ) + + return user_proxy + + def create_keypair(self, username): + """ + Creates a keypair for the user, and + returns it + """ + msg = "create_keypair not implemented" + raise NotImplementedError(msg) + + def delete_keypair(self, username, access_key): + """ + Deletes a keypair from the user and + doesn't return anything + """ + msg = "delete_keypair not implemented" + raise NotImplementedError(msg) + + def add_bucket_acl(self, bucket, username, access=None): + """ + Tries to grant a user access to a bucket + + Args: + bucket (str): Google Bucket Access Group email address. This should + be the address of a Google Group that has read access on a + single bucket. Access is controlled by adding members to this + group. + username (str): An email address of a member to add to the Google + Bucket Access Group. + access (str): IGNORED. For Google buckets, the Google Bucket Access + Group is given access to the bucket through Google's + IAM, so you cannot selectively choose permissions. Once you're + added, you have the access that was set up for that group + in Google IAM. + """ + response = None + with GoogleCloudManager(project_id=self.google_project_id) as g_mgr: + try: + response = g_mgr.add_member_to_group( + member_email=username, group_id=bucket + ) + except Exception as exc: + raise RequestError("Google API Error: {}".format(exc), code=400) + + return response + + def has_bucket_access(self, bucket, user_id): + """ + Check if the user appears in the acl + : returns Bool + """ + msg = "has_bucket_access not implemented" + raise NotImplementedError(msg) + + def list_buckets(self): + """ + Return a list of Bucket objects + : [bucket1, bucket2,...] + """ + msg = "list_buckets not implemented" + raise NotImplementedError(msg) + + def delete_all_keypairs(self, user): + """ + Remove all the keys from a user + : returns None + """ + msg = "delete_all_keypairs not implemented" + raise NotImplementedError(msg) + + def get_bucket(self, bucket): + """ + Return a bucket from the storage + """ + msg = "get_bucket not implemented" + raise NotImplementedError(msg) + + def get_or_create_bucket(self, bucket, access_key=None, secret_key=None): + """ + Tries to retrieve a bucket and if fit fails, + creates and returns one + """ + msg = "get_or_create_bucket not implemented" + raise NotImplementedError(msg) + + def edit_bucket_template(self, template_id, **kwargs): + """ + Change the parameters for the template used to create + the buckets + """ + msg = "edit_bucket_template not implemented" + raise NotImplementedError(msg) + + def update_bucket_acl(self, bucket, user_list): + """ + Add acl's for the list of users + """ + msg = "update_bucket_acl not implemented" + raise NotImplementedError(msg) + + def set_bucket_quota(self, bucket, quota_unit, quota): + """ + Set quota for the entire bucket + """ + msg = "set_bucket_quota not implemented" + raise NotImplementedError(msg) + + def delete_bucket_acl(self, bucket, user): + """ + Set quota for the entire bucket + + Args: + bucket (str): Google Bucket Access Group email address. This should + be the address of a Google Group that has read access on a + single bucket. Access is controlled by adding members to this + group. + user (str): An email address of a member to add to the Google + Bucket Access Group. + """ + response = None + with GoogleCloudManager(project_id=self.google_project_id) as g_mgr: + try: + response = g_mgr.remove_member_from_group( + member_email=user, group_id=bucket + ) + except Exception as exc: + raise RequestError("Google API Error: {}".format(exc), code=400) + + return response diff --git a/fence/scripting/fence_create.py b/fence/scripting/fence_create.py index 91bd5d81e..a513e9a1b 100644 --- a/fence/scripting/fence_create.py +++ b/fence/scripting/fence_create.py @@ -9,9 +9,9 @@ import asyncio from alembic.config import main as alembic_main -from cirrus import GoogleCloudManager -from cirrus.google_cloud.errors import GoogleAuthError -from cirrus.config import config as cirrus_config +from gen3cirrus import GoogleCloudManager +from gen3cirrus.google_cloud.errors import GoogleAuthError +from gen3cirrus.config import config as cirrus_config from cdislogging import get_logger from sqlalchemy import func from userdatamodel.models import ( diff --git a/fence/scripting/google_monitor.py b/fence/scripting/google_monitor.py index e4752212e..ad232519d 100644 --- a/fence/scripting/google_monitor.py +++ b/fence/scripting/google_monitor.py @@ -7,9 +7,9 @@ """ import traceback -from cirrus.google_cloud.iam import GooglePolicyMember -from cirrus import GoogleCloudManager -from cirrus.google_cloud.errors import GoogleAPIError +from gen3cirrus.google_cloud.iam import GooglePolicyMember +from gen3cirrus import GoogleCloudManager +from gen3cirrus.google_cloud.errors import GoogleAPIError from cdislogging import get_logger @@ -357,7 +357,6 @@ def _get_invalid_sa_project_removal_reasons(google_project_validity): def _get_access_removal_reasons(google_project_validity): - removal_reasons = {} if google_project_validity is None: @@ -537,7 +536,6 @@ def _get_users_without_access(db, auth_ids, user_emails, check_linking): no_access = {} for user_email in user_emails: - user = get_user_by_email(user_email, db) or get_user_by_linked_email( user_email, db ) @@ -588,7 +586,6 @@ def _get_users_without_access(db, auth_ids, user_emails, check_linking): def email_user_without_access(user_email, projects, google_project_id): - """ Send email to user, indicating no access to given projects @@ -618,7 +615,6 @@ def email_user_without_access(user_email, projects, google_project_id): def email_users_without_access( db, auth_ids, user_emails, check_linking, google_project_id ): - """ Build list of users without acess and send emails. diff --git a/poetry.lock b/poetry.lock index fc90bc6e4..abe6556cb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -255,17 +255,17 @@ files = [ [[package]] name = "boto3" -version = "1.34.11" +version = "1.34.13" description = "The AWS SDK for Python" optional = false python-versions = ">= 3.8" files = [ - {file = "boto3-1.34.11-py3-none-any.whl", hash = "sha256:1af021e0c6e3040e8de66d403e963566476235bb70f9a8e3f6784813ac2d8026"}, - {file = "boto3-1.34.11.tar.gz", hash = "sha256:31c130a40ec0631059b77d7e87f67ad03ff1685a5b37638ac0c4687026a3259d"}, + {file = "boto3-1.34.13-py3-none-any.whl", hash = "sha256:4c87e2b25a125321394a1bed374293b00bd0e3895e6401a368aa46e1b70df078"}, + {file = "boto3-1.34.13.tar.gz", hash = "sha256:789f65adc1d2cb8e5d36db782e07a733242ca1bd851263af173b61411e32034b"}, ] [package.dependencies] -botocore = ">=1.34.11,<1.35.0" +botocore = ">=1.34.13,<1.35.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.10.0,<0.11.0" @@ -274,13 +274,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.34.11" +version = "1.34.13" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">= 3.8" files = [ - {file = "botocore-1.34.11-py3-none-any.whl", hash = "sha256:1ff1398b6ea670e1c01ac67a33af3da854f8e700d3528289c04f319c330d8250"}, - {file = "botocore-1.34.11.tar.gz", hash = "sha256:51905c3d623c60df5dc5794387de7caf886d350180a01a3dfa762e903edb45a9"}, + {file = "botocore-1.34.13-py3-none-any.whl", hash = "sha256:b39f96e658865bd1f3c2d043794b91cd6206f9db531c0a06b53093ed82d41ef7"}, + {file = "botocore-1.34.13.tar.gz", hash = "sha256:1680b0e0633a546b8d54d1bbd5154e30bb1044d0496e0df7cfd24a383e10b0d3"}, ] [package.dependencies] @@ -318,13 +318,13 @@ files = [ [[package]] name = "cachetools" -version = "4.2.4" +version = "5.3.2" description = "Extensible memoizing collections and decorators" optional = false -python-versions = "~=3.5" +python-versions = ">=3.7" files = [ - {file = "cachetools-4.2.4-py3-none-any.whl", hash = "sha256:92971d3cb7d2a97efff7c7bb1657f21a8f5fb309a37530537c71b1774189f2d1"}, - {file = "cachetools-4.2.4.tar.gz", hash = "sha256:89ea6f1b638d5a73a4f9226be57ac5e4f399d22770b92355f92dcb0f7f001693"}, + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, ] [[package]] @@ -374,7 +374,7 @@ requests = "*" [[package]] name = "cdisutilstest" -version = "0.2.4" +version = "1.1.0" description = "Collection of test data and tools" optional = false python-versions = "*" @@ -384,8 +384,8 @@ develop = false [package.source] type = "git" url = "https://github.com/uc-cdis/cdisutils-test" -reference = "1.0.0" -resolved_reference = "bdfdeb05e45407e839fd954ce6d195d847cd8024" +reference = "feat/cleanup" +resolved_reference = "39795be45da158ee1febe2d796caf17512186b02" [[package]] name = "certifi" @@ -945,22 +945,27 @@ six = ">=1.16.0,<2.0.0" [[package]] name = "gen3cirrus" -version = "2.0.0" +version = "3.0.0" description = "" optional = false -python-versions = "*" -files = [ - {file = "gen3cirrus-2.0.0.tar.gz", hash = "sha256:0bd590c407c42dad5f0b896da0fa30bd01ea6bef5ff7dd11324ec59f14a71793"}, -] +python-versions = "^3.9" +files = [] +develop = false [package.dependencies] -backoff = ">=1.6,<2.0" +backoff = "*" cdislogging = "*" -google-api-python-client = "1.11.0" -google-auth = ">=1.4,<2.0" -google-auth-httplib2 = ">=0.0.3" -google-cloud-storage = ">=1.10,<2.0" -oauth2client = ">=2.0.0,<4.0dev" +google-api-python-client = "*" +google-auth = "*" +google-auth-httplib2 = "*" +google-cloud-storage = "*" +oauth2client = "*" + +[package.source] +type = "git" +url = "https://github.com/uc-cdis/cirrus" +reference = "feat/poetry" +resolved_reference = "0405ec3c25f9d7ac5691a3dbc990352229fa1737" [[package]] name = "gen3config" @@ -996,71 +1001,66 @@ PyYAML = ">=5.1,<6.0" [[package]] name = "google-api-core" -version = "1.32.0" +version = "2.15.0" description = "Google API client core library" optional = false -python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*" +python-versions = ">=3.7" files = [ - {file = "google-api-core-1.32.0.tar.gz", hash = "sha256:101c3c4cf8e7d53badd1dbca7071464353a04b17319a3dbb3a94eaa893da091c"}, - {file = "google_api_core-1.32.0-py2.py3-none-any.whl", hash = "sha256:ead664143b52dccb4f9c8ba865e38d316c5e1c2f8dc1ff5791908c3e0c17ad3f"}, + {file = "google-api-core-2.15.0.tar.gz", hash = "sha256:abc978a72658f14a2df1e5e12532effe40f94f868f6e23d95133bd6abcca35ca"}, + {file = "google_api_core-2.15.0-py3-none-any.whl", hash = "sha256:2aa56d2be495551e66bbff7f729b790546f87d5c90e74781aa77233bcb395a8a"}, ] [package.dependencies] -google-auth = ">=1.25.0,<2.0dev" -googleapis-common-protos = ">=1.6.0,<2.0dev" -packaging = ">=14.3" -protobuf = {version = ">=3.12.0,<4.0.0dev", markers = "python_version > \"3\""} -pytz = "*" -requests = ">=2.18.0,<3.0.0dev" -setuptools = ">=40.3.0" -six = ">=1.13.0" +google-auth = ">=2.14.1,<3.0.dev0" +googleapis-common-protos = ">=1.56.2,<2.0.dev0" +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" +requests = ">=2.18.0,<3.0.0.dev0" [package.extras] -grpc = ["grpcio (>=1.29.0,<2.0dev)"] -grpcgcp = ["grpcio-gcp (>=0.2.2)"] -grpcio-gcp = ["grpcio-gcp (>=0.2.2)"] +grpc = ["grpcio (>=1.33.2,<2.0dev)", "grpcio (>=1.49.1,<2.0dev)", "grpcio-status (>=1.33.2,<2.0.dev0)", "grpcio-status (>=1.49.1,<2.0.dev0)"] +grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] +grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] [[package]] name = "google-api-python-client" -version = "1.11.0" +version = "2.112.0" description = "Google API Client Library for Python" optional = false -python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*" +python-versions = ">=3.7" files = [ - {file = "google-api-python-client-1.11.0.tar.gz", hash = "sha256:caf4015800ef1a18d06d117f47f0219c0c0641f21978f6b1bb5ede7912fab97b"}, - {file = "google_api_python_client-1.11.0-py2.py3-none-any.whl", hash = "sha256:4f596894f702736da84cf89490a810b55ca02a81f0cddeacb3022e2900b11ec6"}, + {file = "google-api-python-client-2.112.0.tar.gz", hash = "sha256:c3bcb5fd70d57f4c94b30c0dbeade53c216febfbf1d771eeb1a2fa74bd0d6756"}, + {file = "google_api_python_client-2.112.0-py2.py3-none-any.whl", hash = "sha256:f5e45d9812376deb7e04cda8d8ca5233aa608038bdbf1253ad8f7edcb7f6d595"}, ] [package.dependencies] -google-api-core = ">=1.18.0,<2dev" -google-auth = ">=1.16.0" -google-auth-httplib2 = ">=0.0.3" -httplib2 = ">=0.9.2,<1dev" -six = ">=1.6.1,<2dev" -uritemplate = ">=3.0.0,<4dev" +google-api-core = ">=1.31.5,<2.0.dev0 || >2.3.0,<3.0.0.dev0" +google-auth = ">=1.19.0,<3.0.0.dev0" +google-auth-httplib2 = ">=0.1.0" +httplib2 = ">=0.15.0,<1.dev0" +uritemplate = ">=3.0.1,<5" [[package]] name = "google-auth" -version = "1.35.0" +version = "2.26.1" description = "Google Authentication Library" optional = false -python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*" +python-versions = ">=3.7" files = [ - {file = "google-auth-1.35.0.tar.gz", hash = "sha256:b7033be9028c188ee30200b204ea00ed82ea1162e8ac1df4aa6ded19a191d88e"}, - {file = "google_auth-1.35.0-py2.py3-none-any.whl", hash = "sha256:997516b42ecb5b63e8d80f5632c1a61dddf41d2a4c2748057837e06e00014258"}, + {file = "google-auth-2.26.1.tar.gz", hash = "sha256:54385acca5c0fbdda510cd8585ba6f3fcb06eeecf8a6ecca39d3ee148b092590"}, + {file = "google_auth-2.26.1-py2.py3-none-any.whl", hash = "sha256:2c8b55e3e564f298122a02ab7b97458ccfcc5617840beb5d0ac757ada92c9780"}, ] [package.dependencies] -cachetools = ">=2.0.0,<5.0" +cachetools = ">=2.0.0,<6.0" pyasn1-modules = ">=0.2.1" -rsa = {version = ">=3.1.4,<5", markers = "python_version >= \"3.6\""} -setuptools = ">=40.3.0" -six = ">=1.9.0" +rsa = ">=3.1.4,<5" [package.extras] -aiohttp = ["aiohttp (>=3.6.2,<4.0.0dev)", "requests (>=2.20.0,<3.0.0dev)"] -pyopenssl = ["pyopenssl (>=20.0.0)"] +aiohttp = ["aiohttp (>=3.6.2,<4.0.0.dev0)", "requests (>=2.20.0,<3.0.0.dev0)"] +enterprise-cert = ["cryptography (==36.0.2)", "pyopenssl (==22.0.0)"] +pyopenssl = ["cryptography (>=38.0.3)", "pyopenssl (>=20.0.0)"] reauth = ["pyu2f (>=0.1.5)"] +requests = ["requests (>=2.20.0,<3.0.0.dev0)"] [[package]] name = "google-auth-httplib2" @@ -1097,23 +1097,25 @@ grpc = ["grpcio (>=1.38.0,<2.0dev)", "grpcio-status (>=1.38.0,<2.0.dev0)"] [[package]] name = "google-cloud-storage" -version = "1.44.0" +version = "2.14.0" description = "Google Cloud Storage API client library" optional = false -python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*" +python-versions = ">=3.7" files = [ - {file = "google-cloud-storage-1.44.0.tar.gz", hash = "sha256:29edbfeedd157d853049302bf5d104055c6f0cb7ef283537da3ce3f730073001"}, - {file = "google_cloud_storage-1.44.0-py2.py3-none-any.whl", hash = "sha256:cd4a223e9c18d771721a85c98a9c01b97d257edddff833ba63b7b1f0b9b4d6e9"}, + {file = "google-cloud-storage-2.14.0.tar.gz", hash = "sha256:2d23fcf59b55e7b45336729c148bb1c464468c69d5efbaee30f7201dd90eb97e"}, + {file = "google_cloud_storage-2.14.0-py2.py3-none-any.whl", hash = "sha256:8641243bbf2a2042c16a6399551fbb13f062cbc9a2de38d6c0bb5426962e9dbd"}, ] [package.dependencies] -google-api-core = {version = ">=1.29.0,<3.0dev", markers = "python_version >= \"3.6\""} -google-auth = {version = ">=1.25.0,<3.0dev", markers = "python_version >= \"3.6\""} -google-cloud-core = {version = ">=1.6.0,<3.0dev", markers = "python_version >= \"3.6\""} -google-resumable-media = {version = ">=1.3.0,<3.0dev", markers = "python_version >= \"3.6\""} -protobuf = {version = "*", markers = "python_version >= \"3.6\""} +google-api-core = ">=1.31.5,<2.0.dev0 || >2.3.0,<3.0.0dev" +google-auth = ">=2.23.3,<3.0dev" +google-cloud-core = ">=2.3.0,<3.0dev" +google-crc32c = ">=1.0,<2.0dev" +google-resumable-media = ">=2.6.0" requests = ">=2.18.0,<3.0.0dev" -six = "*" + +[package.extras] +protobuf = ["protobuf (<5.0.0dev)"] [[package]] name = "google-crc32c" @@ -1215,20 +1217,20 @@ requests = ["requests (>=2.18.0,<3.0.0dev)"] [[package]] name = "googleapis-common-protos" -version = "1.56.4" +version = "1.62.0" description = "Common protobufs used in Google APIs" optional = false python-versions = ">=3.7" files = [ - {file = "googleapis-common-protos-1.56.4.tar.gz", hash = "sha256:c25873c47279387cfdcbdafa36149887901d36202cb645a0e4f29686bf6e4417"}, - {file = "googleapis_common_protos-1.56.4-py2.py3-none-any.whl", hash = "sha256:8eb2cbc91b69feaf23e32452a7ae60e791e09967d81d4fcc7fc388182d1bd394"}, + {file = "googleapis-common-protos-1.62.0.tar.gz", hash = "sha256:83f0ece9f94e5672cced82f592d2a5edf527a96ed1794f0bab36d5735c996277"}, + {file = "googleapis_common_protos-1.62.0-py2.py3-none-any.whl", hash = "sha256:4750113612205514f9f6aa4cb00d523a94f3e8c06c5ad2fee466387dc4875f07"}, ] [package.dependencies] -protobuf = ">=3.15.0,<5.0.0dev" +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" [package.extras] -grpc = ["grpcio (>=1.0.0,<2.0.0dev)"] +grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] [[package]] name = "greenlet" @@ -1445,13 +1447,13 @@ i18n = ["Babel (>=2.7)"] [[package]] name = "jmespath" -version = "0.9.2" +version = "1.0.1" description = "JSON Matching Expressions" optional = false -python-versions = "*" +python-versions = ">=3.7" files = [ - {file = "jmespath-0.9.2-py2.py3-none-any.whl", hash = "sha256:3f03b90ac8e0f3ba472e8ebff083e460c89501d8d41979771535efe9a343177e"}, - {file = "jmespath-0.9.2.tar.gz", hash = "sha256:54c441e2e08b23f12d7fa7d8e6761768c47c969e6aed10eead57505ba760aee9"}, + {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, + {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, ] [[package]] @@ -1652,12 +1654,13 @@ server = ["flask"] [[package]] name = "oauth2client" -version = "3.0.0" +version = "4.1.3" description = "OAuth 2.0 client library" optional = false python-versions = "*" files = [ - {file = "oauth2client-3.0.0.tar.gz", hash = "sha256:5b5b056ec6f2304e7920b632885bd157fa71d1a7f3ddd00a43b1541a8d1a2460"}, + {file = "oauth2client-4.1.3-py2.py3-none-any.whl", hash = "sha256:b8a81cc5d60e2d364f0b1b98f958dbd472887acaf1a5b05e21c28c31a2d6d3ac"}, + {file = "oauth2client-4.1.3.tar.gz", hash = "sha256:d486741e451287f69568a4d26d70d9acd73a2bbfa275746c535b4209891cccc6"}, ] [package.dependencies] @@ -1703,13 +1706,13 @@ invoke = ["invoke (>=1.3)"] [[package]] name = "pbr" -version = "2.0.0" +version = "6.0.0" description = "Python Build Reasonableness" optional = false -python-versions = "*" +python-versions = ">=2.6" files = [ - {file = "pbr-2.0.0-py2.py3-none-any.whl", hash = "sha256:d9b69a26a5cb4e3898eb3c5cea54d2ab3332382167f04e30db5e1f54e1945e45"}, - {file = "pbr-2.0.0.tar.gz", hash = "sha256:0ccd2db529afd070df815b1521f01401d43de03941170f8a800e7531faba265d"}, + {file = "pbr-6.0.0-py2.py3-none-any.whl", hash = "sha256:4a7317d5e3b17a3dccb6a8cfe67dab65b20551404c52c8ed41279fa4f0cb4cda"}, + {file = "pbr-6.0.0.tar.gz", hash = "sha256:d1377122a5a00e2f940ee482999518efe16d745d423a670c27773dfbc3c9a7d9"}, ] [[package]] @@ -1743,43 +1746,24 @@ twisted = ["twisted"] [[package]] name = "protobuf" -version = "3.17.3" -description = "Protocol Buffers" +version = "4.25.1" +description = "" optional = false -python-versions = "*" +python-versions = ">=3.8" files = [ - {file = "protobuf-3.17.3-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ab6bb0e270c6c58e7ff4345b3a803cc59dbee19ddf77a4719c5b635f1d547aa8"}, - {file = "protobuf-3.17.3-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:13ee7be3c2d9a5d2b42a1030976f760f28755fcf5863c55b1460fd205e6cd637"}, - {file = "protobuf-3.17.3-cp35-cp35m-macosx_10_9_intel.whl", hash = "sha256:1556a1049ccec58c7855a78d27e5c6e70e95103b32de9142bae0576e9200a1b0"}, - {file = "protobuf-3.17.3-cp35-cp35m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:f0e59430ee953184a703a324b8ec52f571c6c4259d496a19d1cabcdc19dabc62"}, - {file = "protobuf-3.17.3-cp35-cp35m-win32.whl", hash = "sha256:a981222367fb4210a10a929ad5983ae93bd5a050a0824fc35d6371c07b78caf6"}, - {file = "protobuf-3.17.3-cp35-cp35m-win_amd64.whl", hash = "sha256:6d847c59963c03fd7a0cd7c488cadfa10cda4fff34d8bc8cba92935a91b7a037"}, - {file = "protobuf-3.17.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:145ce0af55c4259ca74993ddab3479c78af064002ec8227beb3d944405123c71"}, - {file = "protobuf-3.17.3-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6ce4d8bf0321e7b2d4395e253f8002a1a5ffbcfd7bcc0a6ba46712c07d47d0b4"}, - {file = "protobuf-3.17.3-cp36-cp36m-win32.whl", hash = "sha256:7a4c97961e9e5b03a56f9a6c82742ed55375c4a25f2692b625d4087d02ed31b9"}, - {file = "protobuf-3.17.3-cp36-cp36m-win_amd64.whl", hash = "sha256:a22b3a0dbac6544dacbafd4c5f6a29e389a50e3b193e2c70dae6bbf7930f651d"}, - {file = "protobuf-3.17.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:ffea251f5cd3c0b9b43c7a7a912777e0bc86263436a87c2555242a348817221b"}, - {file = "protobuf-3.17.3-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:9b7a5c1022e0fa0dbde7fd03682d07d14624ad870ae52054849d8960f04bc764"}, - {file = "protobuf-3.17.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:8727ee027157516e2c311f218ebf2260a18088ffb2d29473e82add217d196b1c"}, - {file = "protobuf-3.17.3-cp37-cp37m-win32.whl", hash = "sha256:14c1c9377a7ffbeaccd4722ab0aa900091f52b516ad89c4b0c3bb0a4af903ba5"}, - {file = "protobuf-3.17.3-cp37-cp37m-win_amd64.whl", hash = "sha256:c56c050a947186ba51de4f94ab441d7f04fcd44c56df6e922369cc2e1a92d683"}, - {file = "protobuf-3.17.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2ae692bb6d1992afb6b74348e7bb648a75bb0d3565a3f5eea5bec8f62bd06d87"}, - {file = "protobuf-3.17.3-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:99938f2a2d7ca6563c0ade0c5ca8982264c484fdecf418bd68e880a7ab5730b1"}, - {file = "protobuf-3.17.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6902a1e4b7a319ec611a7345ff81b6b004b36b0d2196ce7a748b3493da3d226d"}, - {file = "protobuf-3.17.3-cp38-cp38-win32.whl", hash = "sha256:59e5cf6b737c3a376932fbfb869043415f7c16a0cf176ab30a5bbc419cd709c1"}, - {file = "protobuf-3.17.3-cp38-cp38-win_amd64.whl", hash = "sha256:ebcb546f10069b56dc2e3da35e003a02076aaa377caf8530fe9789570984a8d2"}, - {file = "protobuf-3.17.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4ffbd23640bb7403574f7aff8368e2aeb2ec9a5c6306580be48ac59a6bac8bde"}, - {file = "protobuf-3.17.3-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:26010f693b675ff5a1d0e1bdb17689b8b716a18709113288fead438703d45539"}, - {file = "protobuf-3.17.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:e76d9686e088fece2450dbc7ee905f9be904e427341d289acbe9ad00b78ebd47"}, - {file = "protobuf-3.17.3-cp39-cp39-win32.whl", hash = "sha256:a38bac25f51c93e4be4092c88b2568b9f407c27217d3dd23c7a57fa522a17554"}, - {file = "protobuf-3.17.3-cp39-cp39-win_amd64.whl", hash = "sha256:85d6303e4adade2827e43c2b54114d9a6ea547b671cb63fafd5011dc47d0e13d"}, - {file = "protobuf-3.17.3-py2.py3-none-any.whl", hash = "sha256:2bfb815216a9cd9faec52b16fd2bfa68437a44b67c56bee59bc3926522ecb04e"}, - {file = "protobuf-3.17.3.tar.gz", hash = "sha256:72804ea5eaa9c22a090d2803813e280fb273b62d5ae497aaf3553d141c4fdd7b"}, + {file = "protobuf-4.25.1-cp310-abi3-win32.whl", hash = "sha256:193f50a6ab78a970c9b4f148e7c750cfde64f59815e86f686c22e26b4fe01ce7"}, + {file = "protobuf-4.25.1-cp310-abi3-win_amd64.whl", hash = "sha256:3497c1af9f2526962f09329fd61a36566305e6c72da2590ae0d7d1322818843b"}, + {file = "protobuf-4.25.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:0bf384e75b92c42830c0a679b0cd4d6e2b36ae0cf3dbb1e1dfdda48a244f4bcd"}, + {file = "protobuf-4.25.1-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:0f881b589ff449bf0b931a711926e9ddaad3b35089cc039ce1af50b21a4ae8cb"}, + {file = "protobuf-4.25.1-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:ca37bf6a6d0046272c152eea90d2e4ef34593aaa32e8873fc14c16440f22d4b7"}, + {file = "protobuf-4.25.1-cp38-cp38-win32.whl", hash = "sha256:abc0525ae2689a8000837729eef7883b9391cd6aa7950249dcf5a4ede230d5dd"}, + {file = "protobuf-4.25.1-cp38-cp38-win_amd64.whl", hash = "sha256:1484f9e692091450e7edf418c939e15bfc8fc68856e36ce399aed6889dae8bb0"}, + {file = "protobuf-4.25.1-cp39-cp39-win32.whl", hash = "sha256:8bdbeaddaac52d15c6dce38c71b03038ef7772b977847eb6d374fc86636fa510"}, + {file = "protobuf-4.25.1-cp39-cp39-win_amd64.whl", hash = "sha256:becc576b7e6b553d22cbdf418686ee4daa443d7217999125c045ad56322dda10"}, + {file = "protobuf-4.25.1-py3-none-any.whl", hash = "sha256:a19731d5e83ae4737bb2a089605e636077ac001d18781b3cf489b9546c7c80d6"}, + {file = "protobuf-4.25.1.tar.gz", hash = "sha256:57d65074b4f5baa4ab5da1605c02be90ac20c8b40fb137d6a8df9f416b0d0ce2"}, ] -[package.dependencies] -six = ">=1.9" - [[package]] name = "psycopg2" version = "2.9.9" @@ -2314,34 +2298,6 @@ postgresql-psycopg2cffi = ["psycopg2cffi"] pymysql = ["pymysql", "pymysql (<1)"] sqlcipher = ["sqlcipher3_binary"] -[[package]] -name = "storageclient" -version = "0.1.0" -description = "Python client to interact with ceph/cleversafe/google/aws backend" -optional = false -python-versions = "*" -files = [] -develop = false - -[package.dependencies] -boto = ">=2.36.0" -botocore = ">=1.7" -cdislogging = ">=1.0.0" -gen3cirrus = ">=1.0.0" -jmespath = "0.9.2" -pbr = "2.0.0" -protobuf = ">=3.12.0,<3.18.0" -requests = ">=2.5.2" -s3transfer = "*" -six = ">=1.13.0" -urllib3 = ">=1.26.5" - -[package.source] -type = "git" -url = "https://github.com/uc-cdis/storage-client" -reference = "2.0.0" -resolved_reference = "43f116effcc8f9469cee912dfb6cb64506c4ece1" - [[package]] name = "typing-extensions" version = "4.9.0" @@ -2355,13 +2311,13 @@ files = [ [[package]] name = "uritemplate" -version = "3.0.1" -description = "URI templates" +version = "4.1.1" +description = "Implementation of RFC 6570 URI Templates" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +python-versions = ">=3.6" files = [ - {file = "uritemplate-3.0.1-py2.py3-none-any.whl", hash = "sha256:07620c3f3f8eed1f12600845892b0e036a2420acf513c53f7de0abd911a5894f"}, - {file = "uritemplate-3.0.1.tar.gz", hash = "sha256:5af8ad10cec94f215e3f48112de2022e1d5a37ed427fbd88652fa908f2ab7cae"}, + {file = "uritemplate-4.1.1-py2.py3-none-any.whl", hash = "sha256:830c08b8d99bdd312ea4ead05994a38e8936266f84b9a7878232db50b044e02e"}, + {file = "uritemplate-4.1.1.tar.gz", hash = "sha256:4346edfc5c3b79f694bccd6d6099a322bbeb628dbf2cd86eea55a456ce5124f0"}, ] [[package]] @@ -2553,4 +2509,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.9,<4.0.0" -content-hash = "287d3ed6b4d1ca66e3aa8e9964effcb4fda699bae932fb1ebd578938090c33de" +content-hash = "cc51df1078b86badb36bf4a7e985da852dce33a8b56130321f108b1aa433db1a" diff --git a/pyproject.toml b/pyproject.toml index fb4d9fdd5..cb1c23f24 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,7 @@ flask-cors = ">=3.0.3" flask-restful = ">=0.3.8" email_validator = "^1.1.1" gen3authz = "^1.5.1" -gen3cirrus = "^2.0.0" +gen3cirrus = {git = "https://github.com/uc-cdis/cirrus", branch = "feat/poetry"} gen3config = ">=1.1.0" gen3users = "^0.6.0" idna = "^2.10" # https://github.com/python-poetry/poetry/issues/3555 @@ -57,7 +57,6 @@ pyyaml = "^5.1" requests = ">=2.18.0" retry = "^0.9.2" sqlalchemy = "^1.3.3" -storageclient = {git = "https://github.com/uc-cdis/storage-client", rev = "2.0.0"} userdatamodel = ">=2.4.3" werkzeug = ">=2.2.3,<3.0.0" cachelib = "^0.2.0" @@ -67,7 +66,7 @@ Flask-WTF = "^1.0.0" [tool.poetry.dev-dependencies] addict = "^2.2.1" -cdisutilstest = {git = "https://github.com/uc-cdis/cdisutils-test", rev = "1.0.0"} +cdisutilstest = {git = "https://github.com/uc-cdis/cdisutils-test", branch = "feat/cleanup"} codacy-coverage = "^1.3.11" coveralls = "^2.1.1" mock = "^2.0.0" diff --git a/run.py b/run.py index c75ed627c..913803c78 100644 --- a/run.py +++ b/run.py @@ -23,7 +23,7 @@ if config.get("MOCK_STORAGE"): from mock import patch - from cdisutilstest.code.storage_client_mock import get_client + from tests.storageclient.storage_client_mock import get_client patcher = patch("fence.resources.storage.get_client", get_client) patcher.start() diff --git a/tests/conftest.py b/tests/conftest.py index 06edd3d37..b6d07ed19 100755 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,7 +28,6 @@ ) from cryptography.fernet import Fernet import bcrypt -from cdisutilstest.code.storage_client_mock import get_client import jwt from mock import patch, MagicMock, PropertyMock import pytest @@ -53,6 +52,7 @@ from tests import utils from tests.utils import TEST_RAS_USERNAME, TEST_RAS_SUB from tests.utils.oauth2.client import OAuth2TestClient +from tests.storageclient.storage_client_mock import get_client # Allow authlib to use HTTP for local testing. @@ -1571,7 +1571,8 @@ def cloud_manager(): def google_signed_url(): manager = MagicMock() patch( - "fence.blueprints.data.indexd.cirrus.google_cloud.utils.get_signed_url", manager + "fence.blueprints.data.indexd.gen3cirrus.google_cloud.utils.get_signed_url", + manager, ).start() # Note: example outpu/format from google's docs, will not actually work diff --git a/tests/credentials/google/test_credentials.py b/tests/credentials/google/test_credentials.py index a355ac8f8..c32716fe9 100644 --- a/tests/credentials/google/test_credentials.py +++ b/tests/credentials/google/test_credentials.py @@ -17,7 +17,7 @@ ProjectToBucket, StorageAccess, ) -from cdisutilstest.code.storage_client_mock import get_client +from tests.storageclient.storage_client_mock import get_client from fence.config import config diff --git a/tests/data/test_indexed_file.py b/tests/data/test_indexed_file.py index 5988f75fd..b310c8204 100755 --- a/tests/data/test_indexed_file.py +++ b/tests/data/test_indexed_file.py @@ -5,7 +5,7 @@ from unittest import mock from mock import patch -import cirrus +import gen3cirrus import pytest import fence.blueprints.data.indexd as indexd @@ -441,7 +441,7 @@ def test_internal_get_gs_signed_url_cache_new_key_if_old_key_expired( return_value=(sa_private_key), ): with mock.patch.object( - cirrus.google_cloud.utils, + gen3cirrus.google_cloud.utils, "get_signed_url", return_value="https://cloud.google.com/compute/url", ): @@ -514,7 +514,7 @@ def test_internal_get_gs_signed_url_clear_cache_and_parse_json( return_value=(sa_private_key), ): with mock.patch.object( - cirrus.google_cloud.utils, + gen3cirrus.google_cloud.utils, "get_signed_url", return_value="https://cloud.google.com/compute/url", ): @@ -670,7 +670,7 @@ def delete_blob(self): side_effect=Exception("url not available"), ): with patch( - "cirrus.GoogleCloudManager.delete_data_file", + "gen3cirrus.GoogleCloudManager.delete_data_file", side_effect=Exception("url not available"), ): with patch( @@ -737,7 +737,8 @@ def delete_blob(self): return_value=("", 204), ): with patch( - "cirrus.GoogleCloudManager.delete_data_file", return_value=("", 204) + "gen3cirrus.GoogleCloudManager.delete_data_file", + return_value=("", 204), ): with patch( "fence.blueprints.data.indexd.BlobServiceClient.from_connection_string", @@ -787,7 +788,7 @@ def from_connection_string(cls, container_name, blob_name): side_effect=ValueError("Invalid connection string"), ): with patch( - "cirrus.GoogleCloudManager.delete_data_file", + "gen3cirrus.GoogleCloudManager.delete_data_file", side_effect=ValueError("Invalid connection string"), ): with patch( diff --git a/tests/dbgap_sync/conftest.py b/tests/dbgap_sync/conftest.py index 48907ba5b..428fd2639 100644 --- a/tests/dbgap_sync/conftest.py +++ b/tests/dbgap_sync/conftest.py @@ -7,8 +7,11 @@ from yaml import safe_load as yaml_load from cdislogging import get_logger -from cirrus import GoogleCloudManager -from cdisutilstest.code.storage_client_mock import get_client, StorageClientMocker +from gen3cirrus import GoogleCloudManager +from tests.storageclient.storage_client_mock import ( + get_client, + StorageClientMocker, +) import pytest from userdatamodel import Base from userdatamodel.models import * diff --git a/tests/google/test_access_utils.py b/tests/google/test_access_utils.py index e7d1f9269..0c8b6f4f7 100644 --- a/tests/google/test_access_utils.py +++ b/tests/google/test_access_utils.py @@ -4,8 +4,8 @@ from unittest.mock import MagicMock, patch from sqlalchemy import or_ -from cirrus.errors import CirrusError -from cirrus.google_cloud.iam import GooglePolicyMember +from gen3cirrus.errors import CirrusError +from gen3cirrus.google_cloud.iam import GooglePolicyMember import fence from fence.errors import NotFound diff --git a/tests/scripting/test_fence-create.py b/tests/scripting/test_fence-create.py index 29090837f..a6be9319a 100644 --- a/tests/scripting/test_fence-create.py +++ b/tests/scripting/test_fence-create.py @@ -5,8 +5,8 @@ from unittest.mock import MagicMock, patch import pytest -import cirrus -from cirrus.google_cloud.errors import GoogleAuthError +import gen3cirrus +from gen3cirrus.google_cloud.errors import GoogleAuthError from userdatamodel.models import Group from fence.config import config @@ -670,7 +670,6 @@ def test_create_user_access_token( def test_create_refresh_token_with_found_user( app, db_session, oauth_test_client, kid, rsa_private_key ): - DB = config["DB"] username = "test_user" BASE_URL = config["BASE_URL"] @@ -818,7 +817,7 @@ def test_delete_expired_service_accounts_with_one_fail_first( import fence fence.settings = MagicMock() - cirrus.config.update = MagicMock() + gen3cirrus.config.update = MagicMock() cloud_manager.return_value.__enter__.return_value.remove_member_from_group.side_effect = [ HttpError(mock.Mock(status=403), bytes("Permission denied", "utf-8")), {}, @@ -1129,7 +1128,7 @@ def test_delete_expired_google_access_with_one_fail_first( import fence fence.settings = MagicMock() - cirrus.config.update = MagicMock() + gen3cirrus.config.update = MagicMock() cloud_manager.return_value.__enter__.return_value.remove_member_from_group.side_effect = [ HttpError(mock.Mock(status=403), bytes("Permission denied", "utf-8")), {}, @@ -1391,7 +1390,6 @@ def test_verify_google_service_account_member_not_call_delete_operation( def test_link_external_bucket(app, cloud_manager, db_session): - (cloud_manager.return_value.__enter__.return_value.create_group.return_value) = { "email": "test_bucket_read_gbag@someemail.com" } diff --git a/tests/storageclient/__init__.py b/tests/storageclient/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/storageclient/conftest.py b/tests/storageclient/conftest.py new file mode 100644 index 000000000..26c5cb76d --- /dev/null +++ b/tests/storageclient/conftest.py @@ -0,0 +1,98 @@ +import pytest + +# Python 2 and 3 compatible +try: + from unittest.mock import MagicMock + from unittest.mock import patch +except ImportError: + from mock import MagicMock + from mock import patch + +from cdisutilstest.code.request_mocker import RequestMocker +from cdisutilstest.data import ( + createAccount, + cred, + deleteAccount, + editAccountAccessKey, + editAccount, + editVault, + editVaultTemplate, + listAccounts, + listVaults, + viewSystem, +) +from fence.resources.storage.storageclient.google import GoogleCloudStorageClient + + +@pytest.fixture +def google_cloud_storage_client(): + cred.credentials.update({"google_project_id": "test-google-project"}) + return GoogleCloudStorageClient(cred.credentials) + + +@pytest.fixture(scope="function") +def test_cloud_manager(): + manager = MagicMock() + + def mocked_get_group(username): + response = {} + if username == "user0": + response = {"email": "user0_proxy_group@example.com"} + return response + + def mocked_add_member_to_group(member_email, group_id): + response = {} + if ( + group_id == "test_bucket" + and member_email == "user0_proxy_group@example.com" + ): + response = {"email": "user0_proxy_group@example.com"} + else: + raise Exception("cannot add {} to group {}".format(member_email, group_id)) + return response + + def mocked_remove_member_from_group(member_email, group_id): + if ( + group_id == "test_bucket" + and member_email == "user0_proxy_group@example.com" + ): + return {} + else: + raise Exception( + "cannot remove {} from group {}".format(member_email, group_id) + ) + + manager.return_value.__enter__.return_value.get_group = mocked_get_group + manager.return_value.__enter__.return_value.add_member_to_group = ( + mocked_add_member_to_group + ) + manager.return_value.__enter__.return_value.remove_member_from_group = ( + mocked_remove_member_from_group + ) + + patch( + "fence.resources.storage.storageclient.google.GoogleCloudManager", manager + ).start() + return manager + + +@pytest.fixture +def request_mocker(): + files = { + "createAccount": createAccount.values, + "deleteAccount": deleteAccount.values, + "editAccountAccessKey": editAccountAccessKey.values, + "editAccount": editAccount.values, + "editVault": editVault.values, + "editVaultTemplate": editVaultTemplate.values, + "listAccounts": listAccounts.values, + "listVaults": listVaults.values, + "viewSystem": viewSystem.values, + } + req_mock = RequestMocker(files) + patcher = patch("requests.request", req_mock.fake_request) + patcher.start() + + yield req_mock + + patcher.stop() diff --git a/tests/storageclient/storage_client_mock.py b/tests/storageclient/storage_client_mock.py new file mode 100644 index 000000000..5867d13da --- /dev/null +++ b/tests/storageclient/storage_client_mock.py @@ -0,0 +1,225 @@ +""" +This module provides the necessary methods +for mocking the module +userapi.resources.storageclient.__init__.py +modules operations +""" +import unittest, random, string +from mock import patch +from fence.resources.storage.storageclient.base import User, Bucket +from fence.resources.storage.storageclient.errors import NotFoundError, RequestError + + +def get_client(config, backend): + if backend in ["cleversafe", "google"]: + return StorageClientMocker(backend) + else: + raise NotImplementedError() + + +class StorageClientMocker(object): + """ + This class will contain the methods and + the state of the mocking object. It is + supposed to be modifiable by the very calls + it is mocking + """ + + def __init__(self, provider, users={}, buckets={}, permisions={}): + """ + users = {'Name1': User1, 'Name2': User2...} + buckets = {'Name1': Bucket1, 'Name2': Bucket2...} + """ + self.users = users + self.buckets = buckets + self.provider = provider + self.user_counter = 0 + self.bucket_counter = 0 + + def provider(self): + """ + Returns whatever is set up on the attribute + """ + return self.provider + + def list_users(self): + """ + Returns the list of users that + we have created + """ + return self.users.values() + + def has_bucket_access(self, bucket, username): + """ + Check permissions on a user and a bucket + """ + try: + return bucket in self.users[username].permissions.keys() + except KeyError: + raise NotFoundError("User not found") + + def get_user(self, name): + """ + Tries to retrieve a user from the dict + """ + return self.users.get(name) + + def list_bucket(self, backend): + """ + Returns the list of users + """ + return self.buckets.values() + + def create_user(self, name): + """ + Create and return a new user + and add it to the dict + """ + if not name in self.users.keys(): + new_user = User(name) + self.users[name] = new_user + return new_user + else: + raise RequestError("User already exists", 400) + + def delete_user(self, name): + """ + Removes a user from the list + """ + if name in self.users.keys(): + del self.users[name] + else: + raise NotFoundError("User doesn't exists") + + def delete_keypair(self, name, access_key): + """ + Delete the keypair from the user + """ + try: + the_user = self.users[name] + the_user.keys = [ + key for key in the_user.keys if key["access_key"] != access_key + ] + except KeyError as e: + raise e + + def delete_all_keypairs(self, name): + """ + Deletes all keypairs from a user + """ + try: + self.users[name].keys = [] + except KeyError: + raise NotFoundError("The user doesn't exist") + + def create_keypair(self, name): + """ + Create a fake keypair for a user + """ + try: + the_user = self.users[name] + access_key = "".join( + random.choice(string.ascii_uppercase + string.digits) for _ in range(8) + ) + secret_key = "".join( + random.choice(string.ascii_uppercase + string.digits) for _ in range(16) + ) + new_key = {"access_key": access_key, "secret_key": secret_key} + the_user.keys.append(new_key) + return new_key + except KeyError as e: + raise e + + def get_bucket(self, name): + """ + Retrieve a bucket from the list + """ + try: + return self.buckets[name] + except KeyError: + raise NotFoundError("Bucket not found") + + def get_or_create_user(self, name): + """ + Try to get a user and if it fails + creates a new one + """ + return self.get_user(name) or self.create_user(name) + + def get_or_create_bucket(self, name): + """ + Tries to get a bucket and if it fails + creates a new one + """ + try: + return self.get_bucket(name) + except NotFoundError: + mock_key = "XXXXXXXXXX" + mock_secret = "YYYYYYYYYYYYYYYYYY" + return self.create_bucket(name, mock_key, mock_secret) + + def create_bucket(self, name, access_key=None, secret_key=None): + """ + Create a user and insert it in our dictionary + """ + if not name in self.buckets.keys(): + self.bucket_counter += 1 + bucket = Bucket(name, self.bucket_counter, 1024) + self.buckets[name] = bucket + else: + raise RequestError("Bucket name already exists", 400) + + def edit_bucket_template(self, template_id, **kwargs): + """ + Modifies the template + """ + if template_id == 1: + return None + else: + raise NotFoundError("Template not found") + + def update_bucket_acl(self, bucket, new_grant): + """ + Updates the bucket ACL + """ + if bucket in self.buckets.keys(): + return None + else: + raise RequestError("Bucket not found", 400) + + def set_bucket_quota(self, bucket, quota_unit, quota): + try: + self.buckets[bucket].quota = quota + except KeyError: + raise RequestError("Bucket not found", 400) + + def add_bucket_acl(self, bucket, user, access=None): + if not bucket in self.buckets.keys(): + raise NotFoundError("Bucket not found") + elif not user in self.users.keys(): + raise NotFoundError("Bucket not found") + else: + self.users[user].permissions[bucket] = access + + def delete_bucket_acl(self, bucket, user): + """ + Remove user's permission from a bucket + Args: + bucket (str): bucket name + user (str): user name + Returns: + None + """ + if not bucket in self.buckets.keys(): + raise NotFoundError("Bucket not found") + elif not user in self.users.keys(): + raise NotFoundError("Bucket not found") + else: + self.users[user].permissions[bucket] = [] + + def delete_bucket(self, bucket_name): + try: + del self.buckets[bucket_name] + return None + except: + return None diff --git a/tests/storageclient/test_cleversafe_api_client.py b/tests/storageclient/test_cleversafe_api_client.py new file mode 100644 index 000000000..2aeb11991 --- /dev/null +++ b/tests/storageclient/test_cleversafe_api_client.py @@ -0,0 +1,326 @@ +""" +Module for mocking and testing of the +cleversafe API client +""" + +import unittest +from os import path, sys + +sys.path.append(path.dirname(path.dirname(path.abspath(__file__)))) +from fence.resources.storage.storageclient.cleversafe import CleversafeClient +import json +from mock import patch +from fence.resources.storage.storageclient.errors import RequestError, NotFoundError +from cdisutilstest.code.request_mocker import RequestMocker +from cdisutilstest.data import ( + createAccount, + cred, + deleteAccount, + editAccountAccessKey, + editAccount, + editVault, + editVaultTemplate, + listAccounts, + listVaults, + viewSystem, +) + + +class CleversafeManagerTests(unittest.TestCase): + """ + The tests will use a fake response + contructed from data stored in files + on the data folder. + """ + + def setUp(self): + files = { + "createAccount": createAccount.values, + "deleteAccount": deleteAccount.values, + "editAccountAccessKey": editAccountAccessKey.values, + "editAccount": editAccount.values, + "editVault": editVault.values, + "editVaultTemplate": editVaultTemplate.values, + "listAccounts": listAccounts.values, + "listVaults": listVaults.values, + "viewSystem": viewSystem.values, + } + self.req_mock = RequestMocker(files) + self.patcher = patch("requests.request", self.req_mock.fake_request) + self.patcher.start() + self.cm = CleversafeClient(cred.credentials) + + def tearDown(self): + self.patcher.stop() + + def test_get_user_success(self): + """ + Successful retrieval of a user + """ + user = self.cm.get_user("ResponseSuccess") + self.assertEqual(user.username, "ResponseSuccess") + self.assertEqual(user.permissions, {"testVaultName": "owner"}) + self.assertEqual(user.keys[0]["access_key"], "XXXXXXXXXXXXXXXXXXXXXX") + self.assertEqual(user.keys[0]["secret_key"], "YYYYYYYYYYYYYYYYYYYYYYYYYYYYY") + self.assertEqual(user.id, 72) + + def test_get_user_inexistent_user(self): + """ + Retrieval of a nonexistent user + """ + user = self.cm.get_user("NonExistent") + self.assertEqual(user, None) + + def test_get_bucket_by_id_success(self): + """ + Successful retrieval of a vault + """ + response = self.cm._get_bucket_by_id(274) + vault = json.loads(response.text) + self.assertEqual(vault["responseData"]["vaults"][0]["id"], 274) + + def test_list_buckets_success(self): + """ + Successful retrieval of all buckets + """ + vault_list = self.cm.list_buckets() + self.assertEqual(vault_list[0].id, 1) + self.assertEqual(vault_list[1].id, 2) + self.assertEqual(vault_list[2].id, 274) + self.assertEqual(vault_list[3].id, 3) + self.assertEqual(vault_list[0].name, "Testforreal") + self.assertEqual(vault_list[1].name, "whateverName") + self.assertEqual(vault_list[2].name, "testVaultName") + self.assertEqual(vault_list[3].name, "testdata3") + + def test_list_users_success(self): + """ + Successful retrieval of all users from the database + in the form of a list of User objects + """ + user_list = self.cm.list_users() + self.assertEqual(user_list[0].id, 72) + self.assertEqual(user_list[1].id, 1) + self.assertEqual(user_list[2].id, 95) + + def test_create_user_success(self): + """ + Successful creation of a user + """ + user = self.cm.create_user("testUserToBeDeleted") + self.assertEqual(user.id, 72) + self.assertEqual(user.keys[0]["access_key"], "XXXXXXXXXXXXXXXXXXXXXX") + + def test_delete_user_success(self): + """ + Successful deletion of a user + """ + response = self.cm.delete_user("ResponseSuccess") + self.assertEqual(response, None) + + def test_create_keypair_success(self): + """ + Successful creation of a key for a specific user + """ + keypair = self.cm.create_keypair("KeyPairUser") + self.assertEqual( + keypair, + { + "access_key": "XXXXXXXXXXXXXX", + "secret_key": "AAAAAAAAAAAAAHHHHHHHHHHHHHHHHHHHNNNNNN", + }, + ) + + def test_delete_keypair_success(self): + """ + Successful deletion of a key + """ + response = self.cm.delete_keypair("KeyPairUser", "XXXXXXXXXXXXXX") + self.assertEqual(response, None) + + def test_delete_keypair_inexistent_key(self): + """ + Removal of an inexistent key + """ + with self.assertRaises(RequestError): + self.cm.delete_keypair("KeyPairUser", "YYYYYYYYYYYYYYY") + + def test_set_bucket_quota_succes(self): + """ + Successful change of a bucket quota + """ + response = self.cm.set_bucket_quota("Testforreal", "TB", "1") + self.assertEqual(response.status_code, 200) + + def test_set_bucket_quota_error_response(self): + """ + Set bucket quota with error response + """ + with self.assertRaises(RequestError): + self.cm.set_bucket_quota("whateverName", "TB", "1") + + def test_list_users_error_response(self): + """ + List users with error response + """ + self.patcher.stop() + self.patcher = patch( + "requests.request", self.req_mock.fake_request_only_failure + ) + self.patcher.start() + with self.assertRaises(RequestError): + self.cm.get_user("ResponseError") + + def test_get_user_error_response(self): + """ + Get user with error response + """ + with self.assertRaises(RequestError): + self.cm.get_user("ResponseError") + + def test_delete_keypair_error_response(self): + """ + Remove key with error response + """ + with self.assertRaises(RequestError): + self.cm.delete_keypair("KeyPairUser", "YYYYYYYYYYYYYYY") + + def test_delete_all_keypairs_success(self): + """ + Remove all keys success + """ + response = self.cm.delete_all_keypairs("KeyPairUser") + self.assertEqual(response, None) + + def test_delete_all_keypairs_response_error(self): + """ + Remove all keys with response error + """ + with self.assertRaises(RequestError): + self.cm.delete_all_keypairs("KeyPairErrorUser") + + def test_create_keypair_response_error(self): + """ + Key creation with response error + """ + with self.assertRaises(RequestError): + self.cm.create_keypair("KeyPairCreationUser") + + def test_edit_bucket_template_error_response(self): + """ + Edit bucket template with error response + """ + with self.assertRaises(RequestError): + self.cm.edit_bucket_template("0") + + def test_edit_bucket_template_success(self): + """ + Successful modification of the default template + """ + response = self.cm.edit_bucket_template("5") + self.assertEqual(response.status_code, 200) + + def test_delete_user_inexistent_user(self): + """ + Deletion of a inexistent user + WARNING the curl command does not print + anything + """ + response = self.cm.delete_user("KeyPairUser") + self.assertEqual(response, None) + + def test_list_buckets_response_error(self): + """ + List buckets with response error + """ + self.patcher.stop() + self.patcher = patch( + "requests.request", self.req_mock.fake_request_only_failure + ) + self.patcher.start() + with self.assertRaises(RequestError): + self.cm.list_buckets() + + def test_create_user_response_error(self): + """ + Create user with response error + """ + with self.assertRaises(RequestError): + self.cm.create_user("ErroredUser") + + def test_add_bucket_acl_user_not_found_error(self): + """ + ACL addition to bucket with user not found + """ + with self.assertRaises(NotFoundError): + self.cm.add_bucket_acl("whateverName", "NotExistentName", "read-storage") + + def test_add_bucket_acl_bucket_not_found_error(self): + """ + ACL addition to bucket with bucket not found + """ + with self.assertRaises(NotFoundError): + self.cm.add_bucket_acl("NonExistent", "ResponseSuccess", "read-storage") + + def test_add_bucket_acl_success(self): + """ + Successful addition of ACL to bucket + """ + response = self.cm.add_bucket_acl( + "whateverName", "ResponseSuccess", ["read-storage"] + ) + self.assertEqual(response, None) + + def test_get_bucket_success(self): + """ + Successful retrieval of a bucket + """ + bucket = self.cm.get_bucket("testVaultName") + self.assertEqual(bucket.name, "testVaultName") + self.assertEqual(bucket.id, 274) + + def test_get_bucket_response_error(self): + """ + Test retrieval of an inexistent bucket + """ + with self.assertRaises(RequestError): + self.cm.get_bucket("InexistentBucket") + + def test_update_bucket_acl_success(self): + """ + Successful change of acl on a bucket + """ + response = self.cm.update_bucket_acl( + "testVaultName", [("ResponseSuccess", ["read-storage"])] + ) + self.assertEqual(response, None) + + def test_update_bucket_acl_error_response(self): + """ + Change of acl on a bucket with error response + """ + with self.assertRaises(RequestError): + self.cm.update_bucket_acl( + "testVaultName", [("KeyPairCreationUser", ["read-storage"])] + ) + + def test_delete_bucket_acl_success(self): + """ + Successful deletion of an acl + """ + response = self.cm.delete_bucket_acl("testVaultName", "ResponseSuccess") + self.assertEqual(response, None) + + def test_delete_bucket_acl_empty_name(self): + """ + Error handling when deleting an empty user from a bucket + """ + with self.assertRaises(RequestError): + self.cm.delete_bucket_acl("testVaultName", "") + + def test_delete_bucket_acl_empty_bucket(self): + """ + Error handling when deleting an empty bucket + """ + with self.assertRaises(RequestError): + self.cm.delete_bucket_acl("", "ResponseSuccess") diff --git a/tests/storageclient/test_google_api_client.py b/tests/storageclient/test_google_api_client.py new file mode 100644 index 000000000..38532b536 --- /dev/null +++ b/tests/storageclient/test_google_api_client.py @@ -0,0 +1,125 @@ +""" +Module for mocking and testing of the +google API client +""" +import pytest + +# Python 2 and 3 compatible +try: + from unittest.mock import MagicMock + from unittest.mock import patch +except ImportError: + from mock import MagicMock + from mock import patch + +from fence.resources.storage.storageclient.errors import RequestError, NotFoundError + + +class TestGoogleCloudStorageClient(object): + def test_client_creation(self, google_cloud_storage_client, test_cloud_manager): + """ + Ensure that a google project id gets populated + """ + assert google_cloud_storage_client.google_project_id == "test-google-project" + + def test_get_user_success(self, google_cloud_storage_client, test_cloud_manager): + """ + Successful retrieval of a user + """ + user_proxy = google_cloud_storage_client.get_user("user0") + assert getattr(user_proxy, "username") + + def test_get_user_nonexistent_user( + self, google_cloud_storage_client, test_cloud_manager + ): + """ + Retrieval of a nonexistent user + """ + user = google_cloud_storage_client.get_user("NonExistent") + assert user is None + + def test_add_bucket_acl_user_error( + self, google_cloud_storage_client, test_cloud_manager + ): + """ + ACL addition to bucket with user not found + """ + with pytest.raises(RequestError): + google_cloud_storage_client.add_bucket_acl( + access=["read-storage"], bucket="test_bucket", username="NonExistent" + ) + + def test_add_bucket_acl_bucket_error( + self, google_cloud_storage_client, test_cloud_manager + ): + """ + ACL addition to bucket with bucket not found + """ + with pytest.raises(RequestError): + google_cloud_storage_client.add_bucket_acl( + access=["read-storage"], + bucket="NonExistent", + username="user0_proxy_group@example.com", + ) + + def test_add_bucket_acl_success( + self, google_cloud_storage_client, test_cloud_manager + ): + """ + Successful addition of ACL to bucket + """ + response = google_cloud_storage_client.add_bucket_acl( + access=["read-storage"], + bucket="test_bucket", + username="user0_proxy_group@example.com", + ) + + # the response should contain the newly added email + assert response.get("email") == "user0_proxy_group@example.com" + + def test_add_bucket_acl_success_access( + self, google_cloud_storage_client, test_cloud_manager + ): + """ + Successful addition of ACL to bucket even when an access level is + supplied (should be ignored for Google) + """ + response = google_cloud_storage_client.add_bucket_acl( + access=["read-storage"], + bucket="test_bucket", + username="user0_proxy_group@example.com", + ) + + # the response should contain the newly added email + assert response.get("email") == "user0_proxy_group@example.com" + + def test_delete_bucket_acl_success( + self, google_cloud_storage_client, test_cloud_manager + ): + """ + Successful deletion of an acl + """ + response = google_cloud_storage_client.delete_bucket_acl( + bucket="test_bucket", user="user0_proxy_group@example.com" + ) + assert not response + + def test_delete_bucket_acl_empty_name( + self, google_cloud_storage_client, test_cloud_manager + ): + """ + Error handling when deleting an empty user from a bucket + """ + with pytest.raises(RequestError): + google_cloud_storage_client.delete_bucket_acl(bucket="test_bucket", user="") + + def test_delete_bucket_acl_empty_bucket( + self, google_cloud_storage_client, test_cloud_manager + ): + """ + Error handling when deleting an empty bucket + """ + with pytest.raises(RequestError): + google_cloud_storage_client.delete_bucket_acl( + bucket="", user="user0_proxy_group@example.com" + ) diff --git a/tests/storageclient/test_real_storage.py b/tests/storageclient/test_real_storage.py new file mode 100644 index 000000000..36ed13180 --- /dev/null +++ b/tests/storageclient/test_real_storage.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python + +import json +from fence.resources.storage.storageclient import CleversafeClient, errors +import unittest + + +# XXX: tests to fix +import pytest + +pytestmark = pytest.mark.skip + + +class TestStorage(unittest.TestCase): + @classmethod + def setUpClass(self): + with open("cred.json", "r") as f: + self.creds = json.load(f) + self.cc = CleversafeClient(self.creds) + self.test_user = self.cc.create_user("test_suite_user") + self.test_bucket = self.cc.create_bucket( + self.creds["aws_access_key_id"], + self.creds["aws_secret_access_key"], + "test_suite_bucket", + ) + + @classmethod + def tearDownClass(self): + self.cc.delete_user(self.test_user.username) + self.cc.delete_bucket(self.test_bucket.name) + + def setUp(self): + pass + + def tearDown(self): + pass + + def test_create_list_and_delete_bucket(self): + """ + Successful creation, listing and deletion of a vault + """ + new_bucket_name = "my_new_tested_bucket" + self.cc.create_bucket( + self.creds["aws_access_key_id"], + self.creds["aws_secret_access_key"], + new_bucket_name, + ) + bucket = self.cc.get_bucket(new_bucket_name) + self.assertEqual(bucket.name, new_bucket_name) + suite_bucket_found = False + new_bucket_found = False + buckets = self.cc.list_buckets() + for buck in buckets: + if buck.name == new_bucket_name: + new_bucket_found = True + elif buck.name == self.test_bucket.name: + suite_bucket_found = True + self.assertTrue(new_bucket_found) + self.assertTrue(suite_bucket_found) + self.cc.delete_bucket(new_bucket_name) + with self.assertRaises(errors.RequestError): + self.cc.get_bucket(new_bucket_name) + + def test_create_list_and_delete_user(self): + """ + Successful creation, listing and deletion of a user + """ + new_user_name = "my_new_test_user" + self.cc.create_user(new_user_name) + user = self.cc.get_user(new_user_name) + self.assertEqual(user.username, new_user_name) + suite_user_found = False + new_user_found = False + users = self.cc.list_users() + for user in users: + if user.username == new_user_name: + new_user_found = True + elif user.username == self.test_user.username: + suite_user_found = True + self.assertTrue(new_user_found) + self.assertTrue(suite_user_found) + self.cc.delete_user(new_user_name) + user = self.cc.get_user(new_user_name) + self.assertEqual(user, None) + + def test_create_and_delete_keypair_success(self): + """ + Successful creation and deletion of keys + Check that the creation and deletion of + keys work. We check that we keep the same + status that we got at the start + """ + user = self.cc.get_user(self.test_user.username) + original_keys = user.keys + keypair = self.cc.create_keypair(user.username) + user = self.cc.get_user(self.test_user.username) + self.assertIn(keypair, user.keys) + keys = self.cc.delete_keypair(user.username, keypair["access_key"]) + user = self.cc.get_user(self.test_user.username) + self.assertEqual(user.keys, original_keys) + + def test_delete_keypair_inexistent_key(self): + """ + Error handling of inexistent user + """ + with self.assertRaises(errors.RequestError): + self.cc.delete_keypair(self.test_user.username, "inexistent_key") + + def test_set_bucket_quota_succes(self): + """ + Successful change of quota + """ + bucket = self.cc.get_bucket(self.test_bucket.name) + old_quota = bucket.quota + if old_quota != None: + MiB = old_quota / 1048576 + else: + MiB = 1 + self.cc.set_bucket_quota(self.test_bucket.name, "MiB", 2 * MiB) + bucket = self.cc.get_bucket(self.test_bucket.name) + self.assertEqual(bucket.quota / 1048576, MiB * 2) + + def test_delete_all_keypairs_success(self): + """ + Successful deletion of all keypairs + """ + user = self.cc.get_user(self.test_user.username) + original_keys = user.keys + keypair_1 = self.cc.create_keypair(user.username) + keypair_2 = self.cc.create_keypair(user.username) + user = self.cc.get_user(self.test_user.username) + self.assertIn(keypair_1, user.keys) + self.assertIn(keypair_2, user.keys) + keys = self.cc.delete_all_keypairs(user.username) + user = self.cc.get_user(self.test_user.username) + self.assertEqual(user.keys, []) + + def test_edit_bucket_template_success(self): + """ + Successful modification of a template + This method has no way of knowing if the + data on the template has changed, + so once checked once that it works, + it is here only to check that we haven't + broken the template + """ + self.cc.edit_bucket_template(1, description="This is a test description") + self.cc.edit_bucket_template(1, description="") + + def test_delete_user_inexistent_user(self): + """ + Error handling of deletion of an inexistent user + """ + with self.assertRaises(KeyError): + self.cc.delete_user("this_user_will_never_exist") + + def test_add_bucket_acl_user_not_found_error(self): + """ + Error handling of adding a bucket ACL on an inexistent user + """ + with self.assertRaises(errors.NotFoundError): + self.cc.add_bucket_acl( + self.test_bucket.name, "non_existent_user", ["read-storage"] + ) + + def test_add_bucket_acl_bucket_not_found_error(self): + """ + Error handling on adding a bucket ACL on an inexistent bucket + """ + with self.assertRaises(errors.NotFoundError): + self.cc.add_bucket_acl( + "non_existent_bucket", self.test_user.username, ["read-storage"] + ) + + def test_add_bucket_acl_success(self): + """ + Successful addition of an ACL on a bucket + This method has no way of retrieving the information + TODO add writing test on the bucket to check permissions + """ + self.cc.add_bucket_acl( + self.test_bucket.name, self.test_user.username, ["read-storage"] + ) + self.cc.add_bucket_acl( + self.test_bucket.name, self.test_user.username, ["disabled"] + ) + + def test_get_bucket_response_error(self): + """ + Error handling on getting an inexistent bucket + """ + with self.assertRaises(errors.RequestError): + self.cc.get_bucket("inexistent_bucket") + + def test_update_bucket_acl_success(self): + """ + Successful updating of a bucket ACL + This method has no way of retrieving the modified + information + TODO add a writing check + """ + self.cc.update_bucket_acl( + self.test_bucket.name, [(self.test_user.username, ["read-storage"])] + ) + self.cc.update_bucket_acl( + self.test_bucket.name, [(self.test_user.username, ["disabled"])] + )