[apache#2988] feat(client-python): Gravitino python client support ba…

…ckward compatibility (apache#3712)

### What changes were proposed in this pull request?

Ref to: apache#2866 
* Add a check when creating GravitonClient to ensure that the client
version is less than or equal to the server version
* Unlike ObjectMapping in Java, if we add new field in XXXDTO, old
client in Python can still deserialize the object using
`dataclass_json`and ignore the new fields, so we only need to check the
versions.

### Why are the changes needed?

Fix: apache#2988 

### Does this PR introduce _any_ user-facing change?

No

### How was this patch tested?

Add UTs and enable version check in ITs

---------

Co-authored-by: TimWang <tim.wang@pranaq.com>

clients/client-python/.gitignore

@@ -12,6 +12,7 @@ venv
 dist
 build
 README.md
+version.ini
 
 # Unit test / coverage reports
 htmlcov/

clients/client-python/MANIFEST.in

@@ -3,4 +3,5 @@
 
 include requirements.txt
 include requirements-dev.txt
-include README.md
+include README.md
+include version.ini

clients/client-python/build.gradle.kts

@@ -100,14 +100,14 @@ tasks {
   val black by registering(VenvTask::class) {
     dependsOn(pipInstall)
     venvExec = "black"
-    args = listOf("./gravitino", "./tests")
+    args = listOf("./gravitino", "./tests", "./scripts")
   }
 
   val pylint by registering(VenvTask::class) {
     dependsOn(pipInstall)
     mustRunAfter(black)
     venvExec = "pylint"
-    args = listOf("./gravitino", "./tests")
+    args = listOf("./gravitino", "./tests", "./scripts")
   }
 
   val integrationCoverageReport by registering(VenvTask::class){
@@ -162,16 +162,20 @@ tasks {
   }
 
   val build by registering(VenvTask::class) {
+    dependsOn(pylint)
+    venvExec = "python"
+    args = listOf("scripts/generate_version.py")
   }
 
   val distribution by registering(VenvTask::class) {
+    dependsOn(build)
     doFirst {
       delete("README.md")
       generatePypiProjectHomePage()
       delete("dist")
     }
 
-    venvExec = "Python3"
+    venvExec = "python"
     args = listOf("setup.py", "sdist")
 
     doLast {
@@ -206,6 +210,6 @@ tasks {
     it.name.endsWith("envSetup")
   }.all {
     // add install package and code formatting before any tasks
-    finalizedBy(pipInstall, black, pylint)
+    finalizedBy(pipInstall, black, pylint, build)
   }
 }

clients/client-python/gravitino/client/gravitino_client.py

@@ -34,7 +34,7 @@ class GravitinoClient(GravitinoClientBase):
 
     _metalake: GravitinoMetalake
 
-    def __init__(self, uri: str, metalake_name: str):
+    def __init__(self, uri: str, metalake_name: str, check_version: bool = True):
         """Constructs a new GravitinoClient with the given URI, authenticator and AuthDataProvider.
 
         Args:
@@ -45,7 +45,7 @@ def __init__(self, uri: str, metalake_name: str):
         Raises:
             NoSuchMetalakeException if the metalake with specified name does not exist.
         """
-        super().__init__(uri)
+        super().__init__(uri, check_version)
         self._metalake = super().load_metalake(NameIdentifier.of(metalake_name))
 
     def get_metalake(self) -> GravitinoMetalake:

clients/client-python/gravitino/client/gravitino_client_base.py

@@ -4,12 +4,18 @@
 """
 
 import logging
+import configparser
+import os.path
 
 from gravitino.client.gravitino_metalake import GravitinoMetalake
 from gravitino.client.gravitino_version import GravitinoVersion
+from gravitino.dto.version_dto import VersionDTO
 from gravitino.dto.responses.metalake_response import MetalakeResponse
+from gravitino.dto.responses.version_response import VersionResponse
 from gravitino.name_identifier import NameIdentifier
 from gravitino.utils import HTTPClient
+from gravitino.exceptions.gravitino_runtime_exception import GravitinoRuntimeException
+from gravitino.constants.version import VERSION_INI, Version
 
 logger = logging.getLogger(__name__)
 
@@ -29,8 +35,10 @@ class GravitinoClientBase:
     API_METALAKES_IDENTIFIER_PATH = f"{API_METALAKES_LIST_PATH}/"
     """The REST API path prefix for load a specific metalake"""
 
-    def __init__(self, uri: str):
+    def __init__(self, uri: str, check_version: bool = True):
         self._rest_client = HTTPClient(uri)
+        if check_version:
+            self.check_version()
 
     def load_metalake(self, ident: NameIdentifier) -> GravitinoMetalake:
         """Loads a specific Metalake from the Gravitino API.
@@ -57,16 +65,53 @@ def load_metalake(self, ident: NameIdentifier) -> GravitinoMetalake:
 
         return GravitinoMetalake(metalake_response.metalake(), self._rest_client)
 
-    def get_version(self) -> GravitinoVersion:
+    def check_version(self):
+        """Check the compatibility of the client with the target server.
+
+        Raises:
+            GravitinoRuntimeException If the client version is greater than the server version.
+        """
+        server_version = self.get_server_version()
+        client_version = self.get_client_version()
+
+        if client_version > server_version:
+            raise GravitinoRuntimeException(
+                "Gravitino does not support the case that "
+                "the client-side version is higher than the server-side version."
+                f"The client version is {client_version.version()}, and the server version {server_version.version()}"
+            )
+
+    def get_client_version(self) -> GravitinoVersion:
+        """Retrieves the version of the Gravitino Python Client.
+
+        Returns:
+            A GravitinoVersion instance representing the version of the Gravitino Python Client.
+        """
+        config = configparser.ConfigParser()
+
+        if not os.path.exists(VERSION_INI):
+            raise GravitinoRuntimeException(
+                f"Failed to get Gravitino version, version file '{VERSION_INI}' does not exist."
+            )
+        config.read(VERSION_INI)
+
+        version = config["metadata"][Version.VERSION.value]
+        compile_date = config["metadata"][Version.COMPILE_DATE.value]
+        git_commit = config["metadata"][Version.GIT_COMMIT.value]
+
+        return GravitinoVersion(VersionDTO(version, compile_date, git_commit))
+
+    def get_server_version(self) -> GravitinoVersion:
         """Retrieves the version of the Gravitino API.
 
         Returns:
             A GravitinoVersion instance representing the version of the Gravitino API.
         """
         resp = self._rest_client.get("api/version")
-        resp.validate()
+        version_response = VersionResponse.from_json(resp.body, infer_missing=True)
+        version_response.validate()
 
-        return GravitinoVersion(resp.get_version())
+        return GravitinoVersion(version_response.version())
 
     def close(self):
         """Closes the GravitinoClient and releases any underlying resources."""

clients/client-python/gravitino/client/gravitino_version.py

@@ -3,16 +3,68 @@
 This software is licensed under the Apache License version 2.
 """
 
+import re
 from dataclasses import dataclass
+from enum import Enum
 
 from gravitino.dto.version_dto import VersionDTO
+from gravitino.exceptions.gravitino_runtime_exception import GravitinoRuntimeException
+
+
+class Version(Enum):
+    MAJOR = "MAJOR"
+    MINOR = "MINOR"
+    PATCH = "PATCH"
+
+
+VERSION_PATTERN: str = (
+    rf"(?P<{Version.MAJOR.value}>\d+)\.(?P<{Version.MINOR.value}>\d+)\.(?P<{Version.PATCH.value}>\d+)+?"
+)
 
 
 @dataclass
 class GravitinoVersion(VersionDTO):
     """Gravitino version information."""
 
+    major: int
+    minor: int
+    patch: int
+
     def __init__(self, versionDTO):
         super().__init__(
-            versionDTO.version, versionDTO.compile_date, versionDTO.git_commit
+            versionDTO.version(), versionDTO.compile_date(), versionDTO.git_commit()
         )
+
+        m = re.match(VERSION_PATTERN, self.version())
+
+        assert m is not None, "Invalid version string " + self.version()
+
+        self.major = int(m.group(Version.MAJOR.value))
+        self.minor = int(m.group(Version.MINOR.value))
+        self.patch = int(m.group(Version.PATCH.value))
+
+    def __gt__(self, other) -> bool:
+        if not isinstance(other, GravitinoVersion):
+            raise GravitinoRuntimeException(
+                f"{GravitinoVersion.__name__} can't compare with {other.__class__.__name__}"
+            )
+        if self.major > other.major:
+            return True
+        if self.minor > other.minor:
+            return True
+        if self.patch > other.patch:
+            return True
+        return False
+
+    def __eq__(self, other) -> bool:
+        if not isinstance(other, GravitinoVersion):
+            raise GravitinoRuntimeException(
+                f"{GravitinoVersion.__name__} can't compare with {other.__class__.__name__}"
+            )
+        if self.major != other.major:
+            return False
+        if self.minor != other.minor:
+            return False
+        if self.patch != other.patch:
+            return False
+        return True

clients/client-python/gravitino/constants/__init__.py

@@ -0,0 +1,4 @@
+"""
+Copyright 2024 Datastrato Pvt Ltd.
+This software is licensed under the Apache License version 2.
+"""

clients/client-python/gravitino/constants/version.py

@@ -0,0 +1,17 @@
+"""
+Copyright 2024 Datastrato Pvt Ltd.
+This software is licensed under the Apache License version 2.
+"""
+
+from enum import Enum
+from pathlib import Path
+
+PROJECT_HOME = Path(__file__).parent.parent.parent
+VERSION_INI = PROJECT_HOME / "version.ini"
+SETUP_FILE = PROJECT_HOME / "setup.py"
+
+
+class Version(Enum):
+    VERSION = "version"
+    GIT_COMMIT = "gitCommit"
+    COMPILE_DATE = "compileDate"

clients/client-python/gravitino/dto/responses/version_response.py

@@ -0,0 +1,39 @@
+"""
+Copyright 2024 Datastrato Pvt Ltd.
+This software is licensed under the Apache License version 2.
+"""
+
+from dataclasses import dataclass, field
+from dataclasses_json import config
+
+from .base_response import BaseResponse
+from ..version_dto import VersionDTO
+
+
+@dataclass
+class VersionResponse(BaseResponse):
+    """Represents a response containing version of Gravitino."""
+
+    _version: VersionDTO = field(metadata=config(field_name="version"))
+
+    def version(self) -> VersionDTO:
+        return self._version
+
+    def validate(self):
+        """Validates the response data.
+
+        Raise:
+            IllegalArgumentException if name or audit information is not set.
+        """
+        super().validate()
+
+        assert self._version is not None, "version must be non-null"
+        assert (
+            self._version.version() is not None
+        ), "version 'version' must not be null and empty"
+        assert (
+            self._version.compile_date() is not None
+        ), "version 'compile_date' must not be null and empty"
+        assert (
+            self._version.git_commit() is not None
+        ), "version 'git_commit' must not be null and empty"

clients/client-python/gravitino/dto/version_dto.py

@@ -3,18 +3,28 @@
 This software is licensed under the Apache License version 2.
 """
 
-from dataclasses import dataclass
+from dataclasses import dataclass, field
+from dataclasses_json import config
 
 
 @dataclass
 class VersionDTO:
     """Represents a Version Data Transfer Object (DTO)."""
 
-    version: str = ""
+    _version: str = field(metadata=config(field_name="version"))
     """The version of the software."""
 
-    compile_date: str = ""
+    _compile_date: str = field(metadata=config(field_name="compileDate"))
     """The date the software was compiled."""
 
-    git_commit: str = ""
+    _git_commit: str = field(metadata=config(field_name="gitCommit"))
     """The git commit of the software."""
+
+    def version(self) -> str:
+        return self._version
+
+    def compile_date(self) -> str:
+        return self._compile_date
+
+    def git_commit(self) -> str:
+        return self._git_commit

clients/client-python/gravitino/exceptions/gravitino_runtime_exception.py

@@ -8,4 +8,4 @@ class GravitinoRuntimeException(RuntimeError):
     """Base class for all Gravitino runtime exceptions."""
 
     def __init__(self, message, *args):
-        super().__init__(message.format(*args))
+        super().__init__(message % args)

clients/client-python/gravitino/utils/http_client.py

@@ -30,7 +30,7 @@
 
 from gravitino.typing import JSONType
 from gravitino.utils.exceptions import handle_error
-from gravitino.constants import TIMEOUT
+from gravitino.constants.timeout import TIMEOUT
 
 logger = logging.getLogger(__name__)
 

clients/client-python/scripts/__init__.py

@@ -0,0 +1,4 @@
+"""
+Copyright 2024 Datastrato Pvt Ltd.
+This software is licensed under the Apache License version 2.
+"""