feat: new tips for data limit

Author: longxiaofei

app/src/dataSource/index.ts app/src/dataSource/index.tsx

@@ -122,20 +122,47 @@ function initBatchGetDatas(action: string) {
 
 const batchGetDatasBySql = initBatchGetDatas("batch_get_datas_by_sql");
 const batchGetDatasByPayload = initBatchGetDatas("batch_get_datas_by_payload");
+const DEFAULT_LIMIT = 50_000;
+
+function notifyDataLimit() {
+    commonStore.setNotification({
+        type: "warning",
+        title: "Data Limit Reached",
+        message: (<>
+            The current computation has returned more than 50,000 rows of data. 
+            To ensure optimal performance, we are currently rendering only the first 50,000 rows. 
+            If you need to render the entire all datas, please use the 'limit' tool 
+            (<svg className="inline bg-black" width="15" height="15" viewBox="0 0 15 15" fill="none" xmlns="http://www.w3.org/2000/svg">
+                <path
+                    d="M9.94969 7.49989C9.94969 8.85288 8.85288 9.94969 7.49989 9.94969C6.14691 9.94969 5.0501 8.85288 5.0501 7.49989C5.0501 6.14691 6.14691 5.0501 7.49989 5.0501C8.85288 5.0501 9.94969 6.14691 9.94969 7.49989ZM10.8632 8C10.6213 9.64055 9.20764 10.8997 7.49989 10.8997C5.79214 10.8997 4.37847 9.64055 4.13662 8H0.5C0.223858 8 0 7.77614 0 7.5C0 7.22386 0.223858 7 0.5 7H4.13659C4.37835 5.35935 5.79206 4.1001 7.49989 4.1001C9.20772 4.1001 10.6214 5.35935 10.8632 7H14.5C14.7761 7 15 7.22386 15 7.5C15 7.77614 14.7761 8 14.5 8H10.8632Z"
+                    fill="currentColor"
+                    fillRule="evenodd"
+                    clipRule="evenodd"
+                ></path>
+            </svg>) to manually set the maximum number of rows to be returned.
+        </>)
+    }, 60_000);
+}
 
 export function getDatasFromKernelBySql(fieldMetas: any) {
     return async (payload: IDataQueryPayload) => {
         const sql = parser_dsl_with_meta(
             "pygwalker_mid_table",
-            JSON.stringify(payload),
+            JSON.stringify({...payload, limit: payload.limit ?? DEFAULT_LIMIT}),
             JSON.stringify({"pygwalker_mid_table": fieldMetas})
         );
-        const result = await batchGetDatasBySql.getDatas(sql);
-        return (result ?? []) as IRow[];
+        const result = await batchGetDatasBySql.getDatas(sql) ?? [];
+        if (!payload.limit && result.length === DEFAULT_LIMIT) {
+            notifyDataLimit();
+        }
+        return result as IRow[];
     }
 }
 
 export async function getDatasFromKernelByPayload(payload: IDataQueryPayload) {
-    const result = await batchGetDatasByPayload.getDatas(payload);
-    return (result ?? []) as IRow[];
+    const result = await batchGetDatasByPayload.getDatas({...payload, limit: payload.limit ?? DEFAULT_LIMIT}) ?? [];
+    if (!payload.limit && result.length === DEFAULT_LIMIT) {
+        notifyDataLimit();
+    }
+    return result as IRow[];
 }

pygwalker/__init__.py

@@ -10,7 +10,7 @@
 from pygwalker.services.global_var import GlobalVarManager
 from pygwalker.services.kaggle import show_tips_user_kaggle as __show_tips_user_kaggle
 
-__version__ = "0.4.9.2"
+__version__ = "0.4.9.3"
 __hash__ = __rand_str()
 
 from pygwalker.api.jupyter import walk, render, table

pygwalker/api/pygwalker.py

@@ -38,7 +38,6 @@
 from pygwalker.utils.randoms import generate_hash_code
 from pygwalker.communications.hacker_comm import HackerCommunication, BaseCommunication
 from pygwalker._constants import JUPYTER_BYTE_LIMIT, JUPYTER_WIDGETS_BYTE_LIMIT
-from pygwalker.errors import DataCountLimitError
 from pygwalker import __version__
 
 
@@ -404,34 +403,24 @@ def upload_spec_to_cloud(data: Dict[str, Any]):
         def _get_datas(data: Dict[str, Any]):
             sql = data["sql"]
             datas = self.data_parser.get_datas_by_sql(sql)
-            if len(datas) > GlobalVarManager.max_data_length:
-                raise DataCountLimitError()
             return {
                 "datas": datas
             }
 
         def _get_datas_by_payload(data: Dict[str, Any]):
             datas = self.data_parser.get_datas_by_payload(data["payload"])
-            if len(datas) > GlobalVarManager.max_data_length:
-                raise DataCountLimitError()
             return {
                 "datas": datas
             }
 
         def _batch_get_datas_by_sql(data: Dict[str, Any]):
             result = self.data_parser.batch_get_datas_by_sql(data["queryList"])
-            for datas in result:
-                if len(datas) > GlobalVarManager.max_data_length:
-                    raise DataCountLimitError()
             return {
                 "datas": result
             }
 
         def _batch_get_datas_by_payload(data: Dict[str, Any]):
             result = self.data_parser.batch_get_datas_by_payload(data["queryList"])
-            for datas in result:
-                if len(datas) > GlobalVarManager.max_data_length:
-                    raise DataCountLimitError()
             return {
                 "datas": result
             }

pygwalker/errors.py

@@ -43,15 +43,6 @@ class ViewSqlSameColumnError(BaseError):
     pass
 
 
-class DataCountLimitError(BaseError):
-    """Raised when the data count is too large."""
-    def __init__(self) -> None:
-        super().__init__(
-            "The query returned too many data entries, making it difficult for the frontend to render. Please adjust your chart configuration and try again.",
-            code=ErrorCode.UNKNOWN_ERROR
-        )
-
-
 class StreamlitPygwalkerApiError(BaseError):
     """Raised when the config is invalid."""
     def __init__(self) -> None:

pygwalker/services/global_var.py

@@ -1,7 +1,7 @@
 import os
 
 from pandas import DataFrame
-from typing_extensions import Literal
+from typing_extensions import Literal, deprecated
 
 from .config import get_config
 
@@ -46,6 +46,7 @@ def set_last_exported_dataframe(cls, df: DataFrame):
         cls.last_exported_dataframe = df
 
     @classmethod
+    @deprecated("use ui config instead.")
     def set_max_data_length(cls, length: int):
         cls.max_data_length = length