feat: multi project deployments

dagster_uc/config.py

@@ -35,6 +35,7 @@ class UserCodeDeploymentsConfig:
     dagster_gui_url: str | None = None
     verbose: bool = False
     use_az_login: bool = True
+    use_project_name: bool = True
     user_code_deployments_configmap_name: str = "dagster-user-deployments-values-yaml"
     dagster_workspace_yaml_configmap_name: str = "dagster-workspace-yaml"
     uc_deployment_semaphore_name: str = "dagster-uc-semaphore"

dagster_uc/manage_user_code_deployments.py

@@ -167,6 +167,9 @@ def optional_prompt(text: str) -> str | None:
         kubernetes_context=typer.prompt("Kubernetes context of the cluster to use for api calls"),
         dagster_gui_url=optional_prompt("URL of dagster UI"),
         use_az_login=typer.confirm("Whether to use az cli to login to container registry"),
+        use_project_name=typer.confirm(
+            "Whether to use the pyproject.toml project-name as deployment name prefix.",
+        ),
         user_code_deployments_configmap_name=typer.prompt(
             "Configmap name to use for user_code_deployments",
             default="dagster-user-deployments-values-yaml",
@@ -210,6 +213,9 @@ def deployment_revive(
     ],
     tag: Annotated[str, typer.Option("--tag", "-t", help="The tag of the deployment to revive.")],
 ):
+    # In case the UI name separator of the deployment is passed
+    name = name.replace(":", "--")
+
     if not handler._check_deployment_exists(
         name,
     ):
@@ -265,7 +271,13 @@ def deployment_delete(
         typer.echo("\033[1mDeleted all deployments\033[0m")
     else:
         if not name:
-            name = handler.get_deployment_name(deployment_name_suffix="")
+            name = handler.get_deployment_name(
+                deployment_name_suffix="",
+                use_project_name=config.use_project_name,
+            )
+        else:
+            # In case the UI name separator of the deployment is passed
+            name = name.replace(":", "--")
         handler.remove_user_deployment_from_configmap(name)
         handler.delete_k8s_resources_for_user_deployment(
             name,
@@ -296,7 +308,10 @@ def check_deployment(
 ) -> None:
     """This function executes before any other nested cli command is called and loads the configuration object."""
     if not name:
-        name = handler.get_deployment_name()
+        name = handler.get_deployment_name(use_project_name=config.use_project_name)
+    else:
+        # In case the UI name separator of the deployment is passed
+        name = name.replace(":", "--")
     if not handler._check_deployment_exists(name):
         logger.warning(
             f"Deployment with name '{name}' does not seem to exist in environment '{config.environment}'. Attempting to proceed with status check anyways.",
@@ -387,12 +402,13 @@ def is_command_available(command: str) -> bool:
         logger.debug("Using 'podman' to build image.")
         deployment_name = deployment_name or handler.get_deployment_name(
             deployment_name_suffix,
+            use_project_name=config.use_project_name,
         )
         logger.debug("Determining tag...")
         new_tag = gen_tag(
             deployment_name
-            if not handler.config.image_prefix
-            else os.path.join(handler.config.image_prefix, deployment_name),
+            if not config.image_prefix
+            else os.path.join(config.image_prefix, deployment_name),
             config.container_registry,
             config.dagster_version,
             config.use_az_login,
@@ -453,7 +469,7 @@ def is_command_available(command: str) -> bool:
         handler.release_semaphore()
     if config.dagster_gui_url:
         typer.echo(
-            f"Your assets: {config.dagster_gui_url.rstrip('/')}/locations/{deployment_name}/assets\033[0m",
+            f"Your assets: {config.dagster_gui_url.rstrip('/')}/locations/{deployment_name.replace('--', ':')}/assets\033[0m",
         )
     time.sleep(5)
     timeout = 40 if not full_redeploy_done else 240

dagster_uc/uc_handler.py

@@ -125,7 +125,10 @@ def generate_grpc_servers_yaml(servers: list[dict]) -> str:
                 grpc_server = {
                     "host": server["name"],
                     "port": 3030,
-                    "location_name": server["name"],
+                    "location_name": server["name"].replace(
+                        "--",
+                        ":",
+                    ),  ## We replace the -- separator with `:` for more friendly UI name
                 }
                 data["load_from"].append({"grpc_server": grpc_server})
             return yaml.dump(data)
@@ -416,17 +419,30 @@ def _modify_user_deployments(
 
         configmap.patch(new_configmap)  # type: ignore
 
-    def get_deployment_name(self, deployment_name_suffix: str | None = None) -> str:
-        """Creates a deployment name based on the name of the git branch"""
+    def get_deployment_name(  # noqa: D102
+        self,
+        deployment_name_suffix: str | None = None,
+        use_project_name: bool = True,
+    ) -> str:
+        """Creates a deployment name based on the name of the pyproject.toml and name of git branch"""
         logger.debug("Determining deployment name...")
+
+        project_name = self._get_project_name() if use_project_name else None
+
         if not self.config.cicd:
-            name = subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).decode()
+            branch = subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).decode()
             if deployment_name_suffix:
-                name += deployment_name_suffix
+                branch += deployment_name_suffix
+            branch = re.sub(r"[^a-zA-Z0-9]+", "-", branch).strip("-")  # Strips double --
+            name = f"{project_name}--{branch}" if project_name is not None else branch
 
-            return re.sub("[^a-zA-Z0-9-]", "-", name).strip("-")
         else:
-            return f"{self.config.environment}"
+            name = (
+                f"{project_name}--{self.config.environment}"
+                if project_name is not None
+                else self.config.environment
+            )
+        return name
 
     def _ensure_dagster_version_match(self) -> None:
         """Raises an exception if the cluster version of dagster is different than the local version"""
@@ -563,3 +579,16 @@ def release_semaphore(self) -> None:
             logger.debug("patched semaphore to locked: false")
         except Exception as e:
             logger.error(f"Failed to release deployment lock: {e}")
+
+    def _get_project_name(self) -> str | None:
+        import tomli
+
+        try:
+            with open("pyproject.toml", "rb") as fp:
+                data = tomli.load(fp)
+                return re.sub("[^a-zA-Z0-9-]", "-", data["project"]["name"]).strip("-")
+        except FileNotFoundError:
+            logger.warning("""
+            pyproject.toml not found, no project name will be used.
+            Make sure dagster-uc is called in the same directory as the project""")
+            return None

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dagster-uc"
-version = "0.2.4"
+version = "0.3.0"
 authors = [
     {name = "Stefan Verbruggen"}, 
     {name = "Ion Koutsouris"},
@@ -20,6 +20,7 @@ dependencies = [
     "kr8s < 1.0",
     "pyhelm3==0.3.3",
     "typer==0.12.3",
+    "tomli",
     "pyyaml",
     "pytz"
 ]