fix: rendering in streamlit

Author: longxiaofei

pygwalker/__init__.py

@@ -10,7 +10,7 @@
 from pygwalker.services.global_var import GlobalVarManager
 from pygwalker.services.kaggle import show_tips_user_kaggle as __show_tips_user_kaggle
 
-__version__ = "0.4.9.4"
+__version__ = "0.4.9.5"
 __hash__ = __rand_str()
 
 from pygwalker.api.jupyter import walk, render, table

pygwalker/api/streamlit.py

@@ -21,6 +21,7 @@
 from pygwalker.utils.check_walker_params import check_expired_params
 from pygwalker.utils import fallback_value
 from pygwalker.services.streamlit_components import pygwalker_component
+from pygwalker.services.data_parsers import get_dataset_hash
 
 
 class PreFilter(BaseModel):
@@ -65,7 +66,14 @@ def __init__(
         default_tab: Literal["data", "vis"] = "vis",
         **kwargs
     ):
-        """Get pygwalker html render to streamlit
+        """Get pygwalker html render to streamlit.
+        In Streamlit, pygwalker calculates a somewhat inaccurate gid based on the dataset to 
+        distinguish between datasets and uses it as the key for the Streamlit component to
+        avoid redundant rendering.
+
+        In some use case, If user frequently use the same StreamlitRenderer to receive different dataframes,
+        and the differences between these dataframes are so small that pygwalker's gid calculation logic cannot distinguish between different datasets,
+        user should customize method to generate a gid to differentiate between datasets.
 
         Args:
             - dataset (pl.DataFrame | pd.DataFrame | Connector, optional): dataframe.
@@ -87,7 +95,7 @@ def __init__(
         init_streamlit_comm()
 
         self.walker = PygWalker(
-            gid=gid,
+            gid=gid if gid is not None else get_dataset_hash(dataset),
             dataset=dataset,
             field_specs=field_specs if field_specs is not None else [],
             spec=spec,

pygwalker/services/data_parsers.py

@@ -1,58 +1,61 @@
 import sys
-from typing import Dict, Optional, Union, Any, List
+import hashlib
+import pandas as pd
+from typing import Dict, Optional, Union, Any, List, Tuple
+from typing_extensions import Literal
 
 from pygwalker.data_parsers.base import BaseDataParser, FieldSpec
 from pygwalker.data_parsers.database_parser import Connector
 from pygwalker._typing import DataFrame
 
 __classname2method = {}
 
+DatasetType = Literal['pandas', 'polars', 'modin', 'pyspark', 'connector', 'cloud_dataset']
+
 
 # pylint: disable=import-outside-toplevel
-def _get_data_parser(dataset: Union[DataFrame, Connector, str]) -> BaseDataParser:
+def _get_data_parser(dataset: Union[DataFrame, Connector, str]) -> Tuple[BaseDataParser, DatasetType]:
     """
     Get DataFrameDataParser for dataset
     TODO: Maybe you can find a better way to handle the following code
     """
     if type(dataset) in __classname2method:
         return __classname2method[type(dataset)]
 
-    if 'pandas' in sys.modules:
-        import pandas as pd
-        if isinstance(dataset, pd.DataFrame):
-            from pygwalker.data_parsers.pandas_parser import PandasDataFrameDataParser
-            __classname2method[pd.DataFrame] = PandasDataFrameDataParser
-            return __classname2method[pd.DataFrame]
+    if isinstance(dataset, pd.DataFrame):
+        from pygwalker.data_parsers.pandas_parser import PandasDataFrameDataParser
+        __classname2method[pd.DataFrame] = (PandasDataFrameDataParser, "pandas")
+        return __classname2method[pd.DataFrame]
 
     if 'polars' in sys.modules:
         import polars as pl
         if isinstance(dataset, pl.DataFrame):
             from pygwalker.data_parsers.polars_parser import PolarsDataFrameDataParser
-            __classname2method[pl.DataFrame] = PolarsDataFrameDataParser
+            __classname2method[pl.DataFrame] = (PolarsDataFrameDataParser, "polars")
             return __classname2method[pl.DataFrame]
 
     if 'modin.pandas' in sys.modules:
         from modin import pandas as mpd
         if isinstance(dataset, mpd.DataFrame):
             from pygwalker.data_parsers.modin_parser import ModinPandasDataFrameDataParser
-            __classname2method[mpd.DataFrame] = ModinPandasDataFrameDataParser
+            __classname2method[mpd.DataFrame] = (ModinPandasDataFrameDataParser, "modin")
             return __classname2method[mpd.DataFrame]
 
     if 'pyspark' in sys.modules:
         from pyspark.sql import DataFrame as SparkDataFrame
         if isinstance(dataset, SparkDataFrame):
             from pygwalker.data_parsers.spark_parser import SparkDataFrameDataParser
-            __classname2method[SparkDataFrame] = SparkDataFrameDataParser
+            __classname2method[SparkDataFrame] = (SparkDataFrameDataParser, "pyspark")
             return __classname2method[SparkDataFrame]
 
     if isinstance(dataset, Connector):
         from pygwalker.data_parsers.database_parser import DatabaseDataParser
-        __classname2method[DatabaseDataParser] = DatabaseDataParser
+        __classname2method[DatabaseDataParser] = (DatabaseDataParser, "connector")
         return __classname2method[DatabaseDataParser]
 
     if isinstance(dataset, str):
         from pygwalker.data_parsers.cloud_dataset_parser import CloudDatasetParser
-        __classname2method[CloudDatasetParser] = CloudDatasetParser
+        __classname2method[CloudDatasetParser] = (CloudDatasetParser, "cloud_dataset")
         return __classname2method[CloudDatasetParser]
 
     raise TypeError(f"Unsupported data type: {type(dataset)}")
@@ -70,11 +73,45 @@ def get_parser(
     if other_params is None:
         other_params = {}
 
-    parser = _get_data_parser(dataset)(
+    parser_func, _ = _get_data_parser(dataset)
+    parser = parser_func(
         dataset,
         field_specs,
         infer_string_to_date,
         infer_number_to_dimension,
         other_params
     )
     return parser
+
+
+def get_dataset_hash(dataset: Union[DataFrame, Connector, str]) -> str:
+    """Just a less accurate way to get different dataset hash values."""
+    _, dataset_type = _get_data_parser(dataset)
+    if dataset_type in ["pandas", "modin", "polars"]:
+        row_count = dataset.shape[0]
+        other_info = str(dataset.shape) + "_" + dataset_type
+        if row_count > 4000:
+            dataset = dataset[:2000] + dataset[-2000:]
+        if dataset_type == "modin":
+            dataset = dataset._to_pandas()
+        if dataset_type in ["pandas", "modin"]:
+            hash_bytes = pd.util.hash_pandas_object(dataset).values.tobytes() + other_info.encode()
+        else:
+            hash_bytes = dataset.hash_rows().to_numpy().tobytes() + other_info.encode()
+        return hashlib.md5(hash_bytes).hexdigest()
+
+    if dataset_type == "pyspark":
+        shape = ((dataset.count(), len(dataset.columns)))
+        row_count = shape[0]
+        other_info = str(shape) + "_" + dataset_type
+        if row_count > 4000:
+            dataset = dataset.limit(4000)
+        dataset_pd = dataset.toPandas()
+        hash_bytes = pd.util.hash_pandas_object(dataset_pd).values.tobytes() + other_info.encode()
+        return hashlib.md5(hash_bytes).hexdigest()
+
+    if dataset_type == "connector":
+        return hashlib.md5("_".join([dataset.url, dataset.view_sql, dataset_type]).encode()).hexdigest()
+
+    if dataset_type == "cloud_dataset":
+        return hashlib.md5("_".join([dataset, dataset_type]).encode()).hexdigest()