diff --git a/dagster_uc/config.py b/dagster_uc/config.py index 9d5f9d4..3531ca9 100644 --- a/dagster_uc/config.py +++ b/dagster_uc/config.py @@ -35,6 +35,7 @@ class UserCodeDeploymentsConfig: dagster_gui_url: str | None = None verbose: bool = False use_az_login: bool = True + use_project_name: bool = True user_code_deployments_configmap_name: str = "dagster-user-deployments-values-yaml" dagster_workspace_yaml_configmap_name: str = "dagster-workspace-yaml" uc_deployment_semaphore_name: str = "dagster-uc-semaphore" diff --git a/dagster_uc/manage_user_code_deployments.py b/dagster_uc/manage_user_code_deployments.py index 706fbaa..20fc042 100644 --- a/dagster_uc/manage_user_code_deployments.py +++ b/dagster_uc/manage_user_code_deployments.py @@ -167,6 +167,9 @@ def optional_prompt(text: str) -> str | None: kubernetes_context=typer.prompt("Kubernetes context of the cluster to use for api calls"), dagster_gui_url=optional_prompt("URL of dagster UI"), use_az_login=typer.confirm("Whether to use az cli to login to container registry"), + use_project_name=typer.confirm( + "Whether to use the pyproject.toml project-name as deployment name prefix.", + ), user_code_deployments_configmap_name=typer.prompt( "Configmap name to use for user_code_deployments", default="dagster-user-deployments-values-yaml", @@ -210,6 +213,9 @@ def deployment_revive( ], tag: Annotated[str, typer.Option("--tag", "-t", help="The tag of the deployment to revive.")], ): + # In case the UI name separator of the deployment is passed + name = name.replace(":", "--") + if not handler._check_deployment_exists( name, ): @@ -265,7 +271,13 @@ def deployment_delete( typer.echo("\033[1mDeleted all deployments\033[0m") else: if not name: - name = handler.get_deployment_name(deployment_name_suffix="") + name = handler.get_deployment_name( + deployment_name_suffix="", + use_project_name=config.use_project_name, + ) + else: + # In case the UI name separator of the deployment is passed + name = name.replace(":", "--") handler.remove_user_deployment_from_configmap(name) handler.delete_k8s_resources_for_user_deployment( name, @@ -296,7 +308,10 @@ def check_deployment( ) -> None: """This function executes before any other nested cli command is called and loads the configuration object.""" if not name: - name = handler.get_deployment_name() + name = handler.get_deployment_name(use_project_name=config.use_project_name) + else: + # In case the UI name separator of the deployment is passed + name = name.replace(":", "--") if not handler._check_deployment_exists(name): logger.warning( f"Deployment with name '{name}' does not seem to exist in environment '{config.environment}'. Attempting to proceed with status check anyways.", @@ -387,12 +402,13 @@ def is_command_available(command: str) -> bool: logger.debug("Using 'podman' to build image.") deployment_name = deployment_name or handler.get_deployment_name( deployment_name_suffix, + use_project_name=config.use_project_name, ) logger.debug("Determining tag...") new_tag = gen_tag( deployment_name - if not handler.config.image_prefix - else os.path.join(handler.config.image_prefix, deployment_name), + if not config.image_prefix + else os.path.join(config.image_prefix, deployment_name), config.container_registry, config.dagster_version, config.use_az_login, @@ -453,7 +469,7 @@ def is_command_available(command: str) -> bool: handler.release_semaphore() if config.dagster_gui_url: typer.echo( - f"Your assets: {config.dagster_gui_url.rstrip('/')}/locations/{deployment_name}/assets\033[0m", + f"Your assets: {config.dagster_gui_url.rstrip('/')}/locations/{deployment_name.replace('--', ':')}/assets\033[0m", ) time.sleep(5) timeout = 40 if not full_redeploy_done else 240 diff --git a/dagster_uc/uc_handler.py b/dagster_uc/uc_handler.py index d8e04db..1269146 100644 --- a/dagster_uc/uc_handler.py +++ b/dagster_uc/uc_handler.py @@ -125,7 +125,10 @@ def generate_grpc_servers_yaml(servers: list[dict]) -> str: grpc_server = { "host": server["name"], "port": 3030, - "location_name": server["name"], + "location_name": server["name"].replace( + "--", + ":", + ), ## We replace the -- separator with `:` for more friendly UI name } data["load_from"].append({"grpc_server": grpc_server}) return yaml.dump(data) @@ -416,17 +419,30 @@ def _modify_user_deployments( configmap.patch(new_configmap) # type: ignore - def get_deployment_name(self, deployment_name_suffix: str | None = None) -> str: - """Creates a deployment name based on the name of the git branch""" + def get_deployment_name( # noqa: D102 + self, + deployment_name_suffix: str | None = None, + use_project_name: bool = True, + ) -> str: + """Creates a deployment name based on the name of the pyproject.toml and name of git branch""" logger.debug("Determining deployment name...") + + project_name = self._get_project_name() if use_project_name else None + if not self.config.cicd: - name = subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).decode() + branch = subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).decode() if deployment_name_suffix: - name += deployment_name_suffix + branch += deployment_name_suffix + branch = re.sub(r"[^a-zA-Z0-9]+", "-", branch).strip("-") # Strips double -- + name = f"{project_name}--{branch}" if project_name is not None else branch - return re.sub("[^a-zA-Z0-9-]", "-", name).strip("-") else: - return f"{self.config.environment}" + name = ( + f"{project_name}--{self.config.environment}" + if project_name is not None + else self.config.environment + ) + return name def _ensure_dagster_version_match(self) -> None: """Raises an exception if the cluster version of dagster is different than the local version""" @@ -563,3 +579,16 @@ def release_semaphore(self) -> None: logger.debug("patched semaphore to locked: false") except Exception as e: logger.error(f"Failed to release deployment lock: {e}") + + def _get_project_name(self) -> str | None: + import tomli + + try: + with open("pyproject.toml", "rb") as fp: + data = tomli.load(fp) + return re.sub("[^a-zA-Z0-9-]", "-", data["project"]["name"]).strip("-") + except FileNotFoundError: + logger.warning(""" + pyproject.toml not found, no project name will be used. + Make sure dagster-uc is called in the same directory as the project""") + return None diff --git a/pyproject.toml b/pyproject.toml index 1dfdae4..8f2e07d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "dagster-uc" -version = "0.2.4" +version = "0.3.0" authors = [ {name = "Stefan Verbruggen"}, {name = "Ion Koutsouris"}, @@ -20,6 +20,7 @@ dependencies = [ "kr8s < 1.0", "pyhelm3==0.3.3", "typer==0.12.3", + "tomli", "pyyaml", "pytz" ]