diff --git a/app/package.json b/app/package.json index 3b37205e..ade49998 100644 --- a/app/package.json +++ b/app/package.json @@ -17,8 +17,8 @@ "dependencies": { "@headlessui/react": "^1.7.14", "@heroicons/react": "^2.0.8", - "@kanaries/graphic-walker": "0.4.64", - "@kanaries/gw-dsl-parser": "0.1.48-rc5", + "@kanaries/graphic-walker": "0.4.66", + "@kanaries/gw-dsl-parser": "0.1.49", "@radix-ui/react-checkbox": "^1.0.4", "@radix-ui/react-dialog": "^1.0.5", "@radix-ui/react-icons": "^1.3.0", diff --git a/app/yarn.lock b/app/yarn.lock index 82d946d3..1821ea91 100644 --- a/app/yarn.lock +++ b/app/yarn.lock @@ -1045,10 +1045,10 @@ "@jridgewell/resolve-uri" "3.1.0" "@jridgewell/sourcemap-codec" "1.4.14" -"@kanaries/graphic-walker@0.4.64": - version "0.4.64" - resolved "https://registry.yarnpkg.com/@kanaries/graphic-walker/-/graphic-walker-0.4.64.tgz#bc119475ec2e97e69ba2476cabddcbf36e7735db" - integrity sha512-6xm4iXWTllDSrcCSmUIr1IA+XF/hei9sr294E0ezroo5mnHrqX+u/jjAag/OvvbNsDDVvU0xJAD1qWMAdqcYmw== +"@kanaries/graphic-walker@0.4.66": + version "0.4.66" + resolved "https://registry.yarnpkg.com/@kanaries/graphic-walker/-/graphic-walker-0.4.66.tgz#8e0d3d43cd39704a9e7ca781b5e9132c6eb4207d" + integrity sha512-0VmxHhs5tVxVwrXh8AHhK44cnUkdad61dPSFeHnSjHVba0JlBhjnmr4hfm75aZwQlqquE2CRJRBW0zKjBnBlUg== dependencies: "@headlessui-float/react" "^0.11.4" "@headlessui/react" "1.7.12" @@ -1060,6 +1060,7 @@ "@radix-ui/react-context-menu" "^2.1.5" "@radix-ui/react-dialog" "^1.0.5" "@radix-ui/react-dropdown-menu" "^2.0.6" + "@radix-ui/react-hover-card" "^1.0.7" "@radix-ui/react-icons" "^1.3.0" "@radix-ui/react-label" "^2.0.2" "@radix-ui/react-popover" "^1.0.7" @@ -1118,10 +1119,10 @@ vega-lite "^5.6.0" vega-webgl-renderer "^1.0.0-beta.2" -"@kanaries/gw-dsl-parser@0.1.48-rc5": - version "0.1.48-rc5" - resolved "https://registry.yarnpkg.com/@kanaries/gw-dsl-parser/-/gw-dsl-parser-0.1.48-rc5.tgz#b45d92bc85559bc0d477c5a07a9b05d7c2f360eb" - integrity sha512-o+/rE/I1OkK31D+Fj4Uik1czZuTMM7f4BLswmL3yb6bucFxcNZ+bR4s7SDH5dm9L2KVi1A+QLcd0Zh8VaKg9mw== +"@kanaries/gw-dsl-parser@0.1.49": + version "0.1.49" + resolved "https://registry.yarnpkg.com/@kanaries/gw-dsl-parser/-/gw-dsl-parser-0.1.49.tgz#9f5c6c731ca47e52e41c319b31a1f8348d6525f6" + integrity sha512-gK95BVQhO0I7wN7VsntRzwpdTXqeXj6ESdvVTau+oXEYp4SQP2EVkQDLp+bbIKO2VVQ6iRQrqmzGTPC8wL2Xzg== "@kanaries/react-beautiful-dnd@^0.1.1": version "0.1.1" @@ -1409,6 +1410,22 @@ "@radix-ui/react-primitive" "1.0.3" "@radix-ui/react-use-callback-ref" "1.0.1" +"@radix-ui/react-hover-card@^1.0.7": + version "1.0.7" + resolved "https://registry.yarnpkg.com/@radix-ui/react-hover-card/-/react-hover-card-1.0.7.tgz#684bca2504432566357e7157e087051aa3577948" + integrity sha512-OcUN2FU0YpmajD/qkph3XzMcK/NmSk9hGWnjV68p6QiZMgILugusgQwnLSDs3oFSJYGKf3Y49zgFedhGh04k9A== + dependencies: + "@babel/runtime" "^7.13.10" + "@radix-ui/primitive" "1.0.1" + "@radix-ui/react-compose-refs" "1.0.1" + "@radix-ui/react-context" "1.0.1" + "@radix-ui/react-dismissable-layer" "1.0.5" + "@radix-ui/react-popper" "1.1.3" + "@radix-ui/react-portal" "1.0.4" + "@radix-ui/react-presence" "1.0.1" + "@radix-ui/react-primitive" "1.0.3" + "@radix-ui/react-use-controllable-state" "1.0.1" + "@radix-ui/react-icons@^1.3.0": version "1.3.0" resolved "https://registry.npmjs.org/@radix-ui/react-icons/-/react-icons-1.3.0.tgz#c61af8f323d87682c5ca76b856d60c2312dbcb69" diff --git a/pygwalker/__init__.py b/pygwalker/__init__.py index 2eb12624..8893ebb7 100644 --- a/pygwalker/__init__.py +++ b/pygwalker/__init__.py @@ -10,7 +10,7 @@ from pygwalker.services.global_var import GlobalVarManager from pygwalker.services.kaggle import show_tips_user_kaggle as __show_tips_user_kaggle -__version__ = "0.4.8.6" +__version__ = "0.4.8.7" __hash__ = __rand_str() from pygwalker.api.jupyter import walk, render, table diff --git a/pygwalker/data_parsers/database_parser.py b/pygwalker/data_parsers/database_parser.py index d5c9e57c..75778951 100644 --- a/pygwalker/data_parsers/database_parser.py +++ b/pygwalker/data_parsers/database_parser.py @@ -54,8 +54,8 @@ class Connector: "snowflake": {9, 10}, "mysql": {245} } - PRE_SQL_MAP = { - "snowflake": "ALTER SESSION SET STRICT_JSON_OUTPUT=True;", + PRE_INIT_SQL_MAP = { + "snowflake": "ALTER SESSION SET WEEK_OF_YEAR_POLICY=1, WEEK_START=7, STRICT_JSON_OUTPUT=True;", } def __init__(self, url: str, view_sql: str, engine_params: Optional[Dict[str, Any]] = None) -> "Connector": @@ -73,14 +73,16 @@ def _get_engine(self, engine_params: Dict[str, Any]) -> Engine: engine = create_engine(self.url, **engine_params) engine.dialect.requires_name_normalize = False self.engine_map[self.url] = engine + if engine.dialect.name in self.PRE_INIT_SQL_MAP: + pre_init_sql = self.PRE_INIT_SQL_MAP[engine.dialect.name] + with engine.connect(True) as connection: + connection.execute(text(pre_init_sql)) return self.engine_map[self.url] def query_datas(self, sql: str) -> List[Dict[str, Any]]: field_type_map = {} with self.engine.connect() as connection: - if self.dialect_name in self.PRE_SQL_MAP: - connection.execute(text(self.PRE_SQL_MAP[self.dialect_name])) result = connection.execute(text(sql)) if self.dialect_name in self.JSON_TYPE_CODE_SET_MAP: field_type_map = { diff --git a/pygwalker/utils/custom_sqlglot.py b/pygwalker/utils/custom_sqlglot.py index 7d0889f1..b71e4fc4 100644 --- a/pygwalker/utils/custom_sqlglot.py +++ b/pygwalker/utils/custom_sqlglot.py @@ -1,12 +1,15 @@ from sqlglot.dialects.duckdb import DuckDB as DuckdbDialect from sqlglot.dialects.postgres import Postgres as PostgresDialect from sqlglot.dialects.mysql import MySQL as MysqlDialect +from sqlglot.dialects.snowflake import Snowflake as SnowflakeDialect from sqlglot import exp from sqlglot.helper import seq_get from sqlglot.generator import Generator from sqlglot.dialects.dialect import ( build_date_delta, build_date_delta_with_interval, + rename_func, + unit_to_str ) @@ -34,15 +37,75 @@ def _postgres_unix_to_time_sql(self: Generator, expression: exp.UnixToTime) -> s return self.func("to_timestamp", exp.Div(this=timestamp, expression=exp.func("POW", 10, scale))) +# temporary fix for Postgres IN clause(bin filter) +def _postgres_in_sql(self: Generator, expression: exp.In) -> str: + expression.set("expressions", [ + exp.Array(expressions=[ + exp.cast(item, to=exp.DataType.Type.DOUBLE) if isinstance(item, exp.Literal) and item.args.get("is_string") is False else item + for item in in_item_exp.args.get("expressions", []) + ]) if isinstance(in_item_exp, exp.Array) else in_item_exp + for in_item_exp in expression.args.get("expressions", []) + ]) + return self.in_sql(expression) + + +def _postgres_timestamp_trunc(self: Generator, expression: exp.TimestampTrunc) -> str: + if expression.unit.this.lower() == "isoyear": + return self.func("to_date", self.func("to_char", expression.this, exp.Literal.string("IYYY-0001")), exp.Literal.string("IYYY-IDDD")) + + return self.func("DATE_TRUNC", unit_to_str(expression), expression.this) + + +def _postgres_time_to_str_sql(self: Generator, expression: exp.TimeToStr) -> str: + if expression.args.get("format").this == "%U": + # postgres not support non-iso week + # current_pass_days = EXTRACT(isodow FROM DATE_TRUNC('year', date)) + # week_number = floor((EXTRACT(day from date) + current_pass_days - 1) / 7) + return self.sql(exp.Floor( + this=exp.Div( + this=exp.Paren(this=exp.Add( + this=exp.Sub( + this=exp.Cast(this=self.func("TO_CHAR", expression.this, exp.Literal.string("DDD")), to="int"), + expression=exp.Literal.number(1) + ), + expression=exp.Extract(this=exp.Var(this="isodow"), expression=exp.TimestampTrunc(this=expression.this, unit=exp.Literal.string("year"))) + )), + expression=exp.Literal.number(7), + ) + )) + + return self.func("TO_CHAR", expression.this, self.format_time(expression)) + + +def _postgres_str_to_time_sql(self: Generator, expression: exp.StrToTime) -> str: + # adapter duckdb non-iso week + if expression.args.get("format").this == "%Y%U" and isinstance(expression.this, exp.TimeToStr) and expression.this.args.get("format").this == "%Y%U": + return self.sql(exp.Sub( + this=exp.TimestampTrunc(this=expression.this.this, unit=exp.Literal.string("day")), + expression=exp.Mul( + this=exp.Extract(this=exp.Var(this="dow"), expression=expression.this.this), + expression=exp.Interval(this=exp.Literal.number(1), unit=exp.Var(this="day")) + ) + )) + return self.func("TO_TIMESTAMP", expression.this, self.format_time(expression)) + + PostgresDialect.Generator.TRANSFORMS[exp.Round] = lambda _, e: _postgres_round_generator(e) PostgresDialect.Generator.TRANSFORMS[exp.UnixToTime] = _postgres_unix_to_time_sql +PostgresDialect.Generator.TRANSFORMS[exp.In] = _postgres_in_sql +PostgresDialect.Generator.TRANSFORMS[exp.TimestampTrunc] = _postgres_timestamp_trunc +PostgresDialect.Generator.TRANSFORMS[exp.TimeToStr] = _postgres_time_to_str_sql +PostgresDialect.Generator.TRANSFORMS[exp.StrToTime] = _postgres_str_to_time_sql # Mysql Dialect def _mysql_timestamptrunc_sql(self: Generator, expression: exp.TimestampTrunc) -> str: unit = expression.args.get("unit") - start_ts = "'0001-01-01 00:00:00'" + if unit.this.lower() == "isoyear": + unit = exp.Var(this="YEAR") + + start_ts = "'0006-01-01 00:00:00'" timestamp_diff = build_date_delta(exp.TimestampDiff)([unit, start_ts, expression.this]) interval = exp.Interval(this=timestamp_diff, unit=unit) @@ -57,6 +120,13 @@ def _mysql_extract_sql(self: Generator, expression: exp.Extract) -> str: return self.sql(exp.Sub(this=self.func("DAYOFWEEK", expression.expression), expression=exp.Literal.number(1))) if unit == "week": return self.func("WEEK", expression.expression, exp.Literal.number(3)) + if unit == "isoyear": + return self.sql(exp.Floor(this=exp.Div(this=self.func("YEARWEEK", expression.expression, exp.Literal.number(3)), expression=exp.Literal.number(100)))) + if unit == "isodow": + return self.sql(exp.Add( + this=exp.Mod(this=exp.Add(this=self.func("DAYOFWEEK", expression.expression), expression=exp.Literal.number(5)), expression=exp.Literal.number(7)), + expression=exp.Literal.number(1) + )) return self.extract_sql(expression) @@ -67,7 +137,93 @@ def _mysql_unix_to_time_sql(self: Generator, expression: exp.UnixToTime) -> str: return self.func("FROM_UNIXTIME", exp.Div(this=timestamp, expression=exp.func("POW", 10, scale)), self.format_time(expression)) +def _mysql_str_to_time_sql(self: Generator, expression: exp.StrToTime) -> str: + # adapter duckdb non-iso week + if expression.args.get("format").this == "%Y%U" and isinstance(expression.this, exp.TimeToStr) and expression.this.args.get("format").this == "%Y%U": + return _mysql_timestamptrunc_sql(self, exp.TimestampTrunc(this=expression.this.this, unit=exp.Literal.string("WEEK"))) + return self.func("STR_TO_DATE", expression.this, self.format_time(expression)) + + MysqlDialect.Generator.TRANSFORMS[exp.Extract] = _mysql_extract_sql MysqlDialect.Generator.TRANSFORMS[exp.Array] = lambda self, e: self.func("JSON_ARRAY", *e.expressions) MysqlDialect.Generator.TRANSFORMS[exp.TimestampTrunc] = _mysql_timestamptrunc_sql MysqlDialect.Generator.TRANSFORMS[exp.UnixToTime] = _mysql_unix_to_time_sql +MysqlDialect.Generator.TRANSFORMS[exp.Mod] = lambda self, e: self.func("MOD", e.this, e.expression) +MysqlDialect.Generator.TRANSFORMS[exp.StrToTime] = _mysql_str_to_time_sql + + +# Snowflake Dialect +def _snowflake_extract_sql(self: Generator, expression: exp.Extract) -> str: + unit = expression.this.this.lower() + if unit == "isoyear": + return self.func("YEAROFWEEKISO", expression.expression) + if unit == "week": + return self.func("WEEKISO", expression.expression) + if unit == "isodow": + return self.func("DAYOFWEEKISO", expression.expression) + if unit == "dow": + return exp.Sub(this=self.func("DAYOFWEEK", expression.expression), expression=exp.Literal.number(1)) + return rename_func("DATE_PART")(self, expression) + + +def _snowflake_time_to_str(self: Generator, expression: exp.TimeToStr) -> str: + if expression.args.get("format").this == "%U": + # snowflake not support non-iso week + # IFF(TO_CHAR(TO_TIMESTAMP_TZ(TO_CHAR(date, 'YYYY'), 'YYYY'), 'DY') = 'Sun', WEEK(date), WEEK(date)-1) + return self.func( + "IFF", + exp.EQ( + this=self.func("TO_CHAR", self.func("TO_TIMESTAMP_TZ", self.func("TO_CHAR", expression.this, exp.Literal.string('YYYY')), exp.Literal.string('YYYY')), exp.Literal.string("DY")), + expression=exp.Literal.string('Sun') + ), + self.func("WEEK", expression.this), + exp.Sub(this=self.func("WEEK", expression.this), expression=exp.Literal.number(1)) + ) + + return self.func("TO_CHAR", exp.cast(expression.this, exp.DataType.Type.TIMESTAMP), self.format_time(expression)) + + +def _snowflake_str_to_time_sql(self: Generator, expression: exp.StrToTime) -> str: + # adapter duckdb non-iso week + if expression.args.get("format").this == "%Y%U" and isinstance(expression.this, exp.TimeToStr) and expression.this.args.get("format").this == "%Y%U": + return self.func("DATE_TRUNC", exp.Literal.string("WEEK"), expression.this.this) + return self.func("TO_TIMESTAMP", expression.this, self.format_time(expression)) + + +def _snowflake_timestamp_trunc_sql(self: Generator, expression: exp.TimestampTrunc) -> str: + unit = expression.unit.this.lower() + + # dateadd(day, -((date_extract(DAYOFWEEKISO from date)) - 1), date_trunc('day', date)) + trunc_iso_week = self.func( + "dateadd", + exp.Var(this="day"), + exp.Sub( + this=exp.Literal.number(1), + expression=exp.Extract(this=exp.Var(this="DAYOFWEEKISO"), expression=expression.this) + ), + self.func("date_trunc", exp.Literal.string("day"), expression.this) + ) + + # dateadd(week, 1-(WEEKISO(date)), trunc_iso_week) + if unit == "isoyear": + return self.func( + "dateadd", + exp.Var(this="week"), + exp.Sub( + this=exp.Literal.number(1), + expression=self.func("WEEKISO", expression.this) + ), + trunc_iso_week + ) + + # duckdb week means "isoweek" + if unit == "week": + return trunc_iso_week + + return self.func("DATE_TRUNC", expression.unit, expression.this) + + +SnowflakeDialect.Generator.TRANSFORMS[exp.Extract] = _snowflake_extract_sql +SnowflakeDialect.Generator.TRANSFORMS[exp.TimeToStr] = _snowflake_time_to_str +SnowflakeDialect.Generator.TRANSFORMS[exp.StrToTime] = _snowflake_str_to_time_sql +SnowflakeDialect.Generator.TRANSFORMS[exp.TimestampTrunc] = _snowflake_timestamp_trunc_sql diff --git a/pyproject.toml b/pyproject.toml index 61bd29f5..335fd511 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ dependencies = [ "requests", "arrow", "sqlalchemy", - "gw_dsl_parser==0.1.48a6", + "gw_dsl_parser==0.1.49", "appdirs", "segment-analytics-python==2.2.3", "pandas",