From 766f9c686b3e5751565dcdd8beb6dae59d42412c Mon Sep 17 00:00:00 2001 From: Jiri Podivin Date: Wed, 19 Feb 2025 11:13:49 +0100 Subject: [PATCH] Storing all user data collection parameters in config Signed-off-by: Jiri Podivin --- ols/app/models/config.py | 19 ++++++++++++++++- ols/user_data_collection/data_collector.py | 24 +++++++++------------- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/ols/app/models/config.py b/ols/app/models/config.py index 6e77bfad..69112c1c 100644 --- a/ols/app/models/config.py +++ b/ols/app/models/config.py @@ -1050,7 +1050,11 @@ def __eq__(self, other: object) -> bool: class UserDataCollectorConfig(BaseModel): - """User data collection configuration.""" + """User data collection configuration. + + All time information is given in seconds. + + """ data_storage: Optional[DirectoryPath] = None log_level: int = logging.INFO @@ -1058,6 +1062,14 @@ class UserDataCollectorConfig(BaseModel): run_without_initial_wait: bool = False ingress_env: Literal["stage", "prod"] = "prod" cp_offline_token: Optional[str] = None + initial_wait: int = 60 * 5 # 5 minutes in seconds + ingress_timeout: int = 30 + ingress_base_delay: int = 60 # exponential backoff parameter + ingress_max_retries: int = 3 # exponential backoff parameter + access_token_generation_timeout: int = 5 + user_agent: str = ( + "openshift-lightspeed-operator/user-data-collection cluster/{cluster_id}" + ) def __init__(self, **data: Any) -> None: """Initialize configuration.""" @@ -1072,6 +1084,11 @@ def validate_token_is_set_when_needed(self) -> Self: """Check that cp_offline_token is set when env is stage.""" if self.ingress_env == "stage" and self.cp_offline_token is None: raise ValueError("cp_offline_token is required in stage environment") + if "{cluster_id}" not in self.user_agent: + raise ValueError( + "user_agent must contain a {cluster_id} substring, " + "as a placeholder for k8s cluster ID" + ) return self diff --git a/ols/user_data_collection/data_collector.py b/ols/user_data_collection/data_collector.py index ceda670c..735ea86f 100644 --- a/ols/user_data_collection/data_collector.py +++ b/ols/user_data_collection/data_collector.py @@ -32,6 +32,8 @@ DEFAULT_CONFIGURATION_FILE, ) +OLS_USER_DATA_MAX_SIZE = 100 * 1024 * 1024 # 100 MiB + cfg_file = os.environ.get( CONFIGURATION_FILE_NAME_ENV_VARIABLE, DEFAULT_CONFIGURATION_FILE ) @@ -44,14 +46,6 @@ # pylint: disable-next=C0413 from ols.src.auth.k8s import K8sClientSingleton # noqa: E402 -INITIAL_WAIT = 60 * 5 # 5 minutes in seconds -INGRESS_TIMEOUT = 30 # seconds -INGRESS_BASE_DELAY = 60 # exponential backoff parameter -INGRESS_MAX_RETRIES = 3 # exponential backoff parameter -REDHAT_SSO_TIMEOUT = 5 # seconds - -OLS_USER_DATA_MAX_SIZE = 100 * 1024 * 1024 # 100 MiB -USER_AGENT = "openshift-lightspeed-operator/user-data-collection cluster/{cluster_id}" logging.basicConfig( level=udc_config.log_level, stream=sys.stdout, @@ -106,7 +100,9 @@ def access_token_from_offline_token(offline_token: str) -> str: "refresh_token": offline_token, } - response = requests.post(url + endpoint, data=data, timeout=REDHAT_SSO_TIMEOUT) + response = requests.post( + url + endpoint, data=data, timeout=udc_config.access_token_generation_timeout + ) try: if response.status_code == requests.codes.ok: return response.json()["access_token"] @@ -220,7 +216,7 @@ def wrapper(*args: Any, **kwargs: Any) -> None: @exponential_backoff_decorator( - max_retries=INGRESS_MAX_RETRIES, base_delay=INGRESS_BASE_DELAY + max_retries=udc_config.ingress_max_retries, base_delay=udc_config.ingress_base_delay ) def upload_data_to_ingress(tarball: io.BytesIO) -> requests.Response: """Upload the tarball to a Ingress. @@ -254,7 +250,7 @@ def upload_data_to_ingress(tarball: io.BytesIO) -> requests.Response: cluster_id = K8sClientSingleton.get_cluster_id() token = get_cloud_openshift_pull_secret() headers = { - "User-Agent": USER_AGENT.format(cluster_id=cluster_id), + "User-Agent": udc_config.user_agent.format(cluster_id=cluster_id), "Authorization": f"Bearer {token}", } @@ -264,7 +260,7 @@ def upload_data_to_ingress(tarball: io.BytesIO) -> requests.Response: response = s.post( url=url, files=payload, - timeout=INGRESS_TIMEOUT, + timeout=udc_config.ingress_timeout, ) if response.status_code != requests.codes.accepted: @@ -405,9 +401,9 @@ def disabled_by_file() -> bool: if not udc_config.run_without_initial_wait: logger.info( "collection script started, waiting %d seconds before first collection", - INITIAL_WAIT, + udc_config.initial_wait, ) - time.sleep(INITIAL_WAIT) + time.sleep(udc_config.initial_wait) while True: if not disabled_by_file(): gather_ols_user_data(udc_config.data_storage.as_posix())