feat: add global param max_data_length

pygwalker/__init__.py

@@ -10,7 +10,7 @@
 from pygwalker.services.global_var import GlobalVarManager
 from pygwalker.services.kaggle import show_tips_user_kaggle as __show_tips_user_kaggle
 
-__version__ = "0.4.8.3"
+__version__ = "0.4.8.4"
 __hash__ = __rand_str()
 
 from pygwalker.api.jupyter import walk, render, table

pygwalker/api/pygwalker.py

@@ -42,8 +42,6 @@
 from pygwalker.errors import DataCountLimitError
 from pygwalker import __version__
 
-RESPONSE_MAX_DATA_LENGTH = 1 * 1000 * 1000
-
 
 class PygWalker:
     """PygWalker"""
@@ -385,15 +383,15 @@ def upload_spec_to_cloud(data: Dict[str, Any]):
         def _get_datas(data: Dict[str, Any]):
             sql = data["sql"]
             datas = self.data_parser.get_datas_by_sql(sql)
-            if len(datas) > RESPONSE_MAX_DATA_LENGTH:
+            if len(datas) > GlobalVarManager.max_data_length:
                 raise DataCountLimitError()
             return {
                 "datas": datas
             }
 
         def _get_datas_by_payload(data: Dict[str, Any]):
             datas = self.data_parser.get_datas_by_payload(data["payload"])
-            if len(datas) > RESPONSE_MAX_DATA_LENGTH:
+            if len(datas) > GlobalVarManager.max_data_length:
                 raise DataCountLimitError()
             return {
                 "datas": datas
@@ -402,7 +400,7 @@ def _get_datas_by_payload(data: Dict[str, Any]):
         def _batch_get_datas_by_sql(data: Dict[str, Any]):
             result = self.data_parser.batch_get_datas_by_sql(data["queryList"])
             for datas in result:
-                if len(datas) > RESPONSE_MAX_DATA_LENGTH:
+                if len(datas) > GlobalVarManager.max_data_length:
                     raise DataCountLimitError()
             return {
                 "datas": result
@@ -411,7 +409,7 @@ def _batch_get_datas_by_sql(data: Dict[str, Any]):
         def _batch_get_datas_by_payload(data: Dict[str, Any]):
             result = self.data_parser.batch_get_datas_by_payload(data["queryList"])
             for datas in result:
-                if len(datas) > RESPONSE_MAX_DATA_LENGTH:
+                if len(datas) > GlobalVarManager.max_data_length:
                     raise DataCountLimitError()
             return {
                 "datas": result

pygwalker/services/global_var.py

@@ -14,6 +14,7 @@ class GlobalVarManager:
     kanaries_api_host = "https://api.kanaries.net"
     kanaries_main_host = "https://kanaries.net"
     last_exported_dataframe = None
+    max_data_length = 1000 * 1000
 
     @classmethod
     def set_env(cls, env: Literal['Jupyter', 'Streamlit']):
@@ -42,3 +43,7 @@ def set_privacy(cls, privacy: Literal['offline', 'update-only', 'events']):
     @classmethod
     def set_last_exported_dataframe(cls, df: DataFrame):
         cls.last_exported_dataframe = df
+
+    @classmethod
+    def set_max_data_length(cls, length: int):
+        cls.max_data_length = length

tests/stress-test-polars.ipynb

@@ -19,6 +19,30 @@
     "tags": []
    },
    "outputs": [],
+   "source": [
+    "import pygwalker.services.config as pyg_conf\n",
+    "\n",
+    "pyg_conf.set_config({'privacy': 'offline'})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e1a103bc",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!pygwalker config --list"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b944c557-ec0d-4afb-afcb-c5fb2376b25e",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
    "source": [
     "import polars as pl\n",
     "df = pl.scan_csv('./bike_sharing_dc.csv', try_parse_dates = True)\n",
@@ -37,23 +61,13 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "b944c557-ec0d-4afb-afcb-c5fb2376b25e",
-   "metadata": {
-    "tags": []
-   },
+   "id": "527bfe29-c8fd-4051-863f-d1f63e0de4c3",
+   "metadata": {},
    "outputs": [],
    "source": [
     "import pygwalker as pyg\n",
     "pyg.walk(df)"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "527bfe29-c8fd-4051-863f-d1f63e0de4c3",
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {