v1.2.0 - adds record support

kislerdm · May 17, 2020 · a3c4314 · a3c4314
1 parent d3444c1
commit a3c4314
Show file tree

Hide file tree

Showing 7 changed files with 284 additions and 74 deletions.
diff --git a/gbqschema_converter/__init__.py b/gbqschema_converter/__init__.py
@@ -9,5 +9,5 @@
 - https://cloud.google.com/bigquery/docs/schemas#creating_a_json_schema_file
 - https://json-schema.org/
 """
-__version__ = "1.1.0"
+__version__ = "1.2.0"
 __all__ = ['__version__', 'gbqschema_to_jsonschema', 'jsonschema_to_gbqschema']
diff --git a/gbqschema_converter/__main__.py b/gbqschema_converter/__main__.py
@@ -18,12 +18,8 @@
 logs = logging.getLogger(help_string)
 
 
-def get_args():
-    """CL input parameter.
-
-    Returns:
-
-    """
+def get_args() -> argparse.Namespace:
+    """CL input parameters."""
     parser = argparse.ArgumentParser(description=help_string)
     required_either = parser.add_mutually_exclusive_group(required=True)
     required_either.add_argument('-i', '--input', 

diff --git a/gbqschema_converter/gbqschema_to_jsonschema.py b/gbqschema_converter/gbqschema_to_jsonschema.py
@@ -47,6 +47,7 @@
                     "DATETIME",
                     "TIME",
                     "TIMESTAMP",
+                    "RECORD",
                 ],
             },
             "mode": {
@@ -64,7 +65,7 @@
                 ],
             },
         },
-        "additionalProperties": False,
+        "additionalProperties": True,
     },
 }
 
@@ -77,15 +78,7 @@
         "$ref": "#/definitions/element"
     },
     "definitions": {
-        "element": {
-            "type": "object",
-            "properties": {
-
-            },
-            "additionalProperties": False,
-            "required": [
-            ],
-        },
+        "element": {},
     },
 }
 
@@ -112,7 +105,8 @@
         "type": "string",
         "pattern": "^((|[0-1])[0-9]|2[0-3]):((|[0-5])[0-9]):((|[0-5])[0-9])(|.[0-9]{1,6})$"
     },
-    TIMESTAMP={"type": "string", "format": "date-time"}
+    TIMESTAMP={"type": "string", "format": "date-time"},
+    RECORD={"type": "object"},
 )
 
 
@@ -136,26 +130,42 @@ def json_representation(gbq_schema: dict,
 
       fastjsonschema.JsonSchemaException: Error occured if input Google BigQuery schema is invalid.
     """
-    try:
-        validate_json(gbq_schema)
-    except fastjsonschema.JsonSchemaException as ex:
-        raise ex
-
     output = deepcopy(TEMPLATE)
-
-    for element in gbq_schema:
-        key = element['name']
-
-        output['definitions']['element']['properties'][key] = getattr(map_types,
-                                                                      element['type'])
-
-        if 'description' in element:
-            if element['description']:
-                output['definitions']['element']['properties'][key]['description'] = element['description']
-
-        if 'mode' in element:
-            if element['mode'] == "REQUIRED":
-                output['definitions']['element']['required'].append(key)
+
+    def _converter(gbq_schema: dict) -> dict:
+        """Conversion step."""
+        try:
+            validate_json(gbq_schema)
+        except fastjsonschema.JsonSchemaException as ex:
+            raise ex
+
+        output = {
+            "type": "object",
+            "properties": {
+            },
+            "additionalProperties": False,
+            "required": [
+            ],
+        }
+
+        for element in gbq_schema:
+            key = element['name']
+
+            output['properties'][key] = getattr(map_types, element['type'])
+
+            if 'mode' in element:
+                if element['mode'] == "REQUIRED":
+                    output['required'].append(key)
+
+            if element['type'] == "RECORD":
+                output['properties'][key] = _converter(element['fields'])
+
+        if not output['required']:
+            _ = output.pop('required')
+
+        return output
+
+    output['definitions']['element'] = _converter(gbq_schema)
 
     output['definitions']['element']['additionalProperties'] = additional_properties
 
@@ -180,17 +190,33 @@ def sdk_representation(gbq_schema: List[SchemaField],
     """
     output = deepcopy(TEMPLATE)
 
-    for element in gbq_schema:
-        key = element.name
-
-        output['definitions']['element']['properties'][key] = getattr(map_types,
-                                                                      element.field_type)
-
-        if element.description:
-            output['definitions']['element']['properties'][key]['description'] = element.description
-
-        if element.mode == "REQUIRED":
-            output['definitions']['element']['required'].append(key)
+    def _converter(gbq_schema: dict) -> dict:
+        output = {
+            "type": "object",
+            "properties": {
+            },
+            "additionalProperties": False,
+            "required": [
+            ],
+        }
+
+        for element in gbq_schema:
+            key = element.name
+
+            output['properties'][key] = getattr(map_types, element.field_type)
+
+            if element.mode == "REQUIRED":
+                output['required'].append(key)
+
+            if element.field_type == "RECORD":
+                output['properties'][key] = _converter(element.fields)
+
+        if not output['required']:
+            _ = output.pop('required')
+
+        return output
+
+    output['definitions']['element'] = _converter(gbq_schema)
 
     output['definitions']['element']['additionalProperties'] = additional_properties
 

diff --git a/gbqschema_converter/jsonschema_to_gbqschema.py b/gbqschema_converter/jsonschema_to_gbqschema.py
@@ -9,14 +9,15 @@
 
 
 MapTypes = namedtuple("map_types",
-                      ['integer', 'number', 'boolean', 'string', 'date'])
+                      ['integer', 'number', 'boolean', 'string', 'date', 'object'])
 
 map_types = MapTypes(
-    integer="INTEGER",
-    number="NUMERIC",
+    integer="INT64",
+    number="FLOAT64",
     boolean="BOOLEAN",
     string="STRING",
-    date="DATE"
+    date="DATE",
+    object="RECORD"
 )
 
 TEMPLATE_GBQ_COLUMN = {
@@ -82,10 +83,7 @@ def __gbq_columns(properties: dict,
             else:
                 gbq_column['type'] = "TIMESTAMP" if v['format'] == "date-time"\
                     else getattr(map_types, v['format']) if v['format'] in map_types.__dir__()\
-                        else "STRING"
-
-            if to_sdk_schema:
-                gbq_column['field_type'] = gbq_column.pop('type')
+                    else "STRING"
 
             if required:
                 if k in required:
@@ -96,16 +94,20 @@ def __gbq_columns(properties: dict,
             else:
                 _ = gbq_column.pop('description')
 
+            if gbq_column['type'] == "RECORD":
+                gbq_column['fields'] = __gbq_columns(v['properties'],
+                                                     v['required'])
+
             if to_sdk_schema:
+                gbq_column['field_type'] = gbq_column.pop('type')
                 gbq_column = SchemaField(**gbq_column)
 
             output.append(gbq_column)
         return output
 
     output = []
 
-    if 'definitions' in json_schema\
-            or 'items' in json_schema:
+    if 'definitions' in json_schema:
         for prop in json_schema['definitions'].values():
             properties = prop['properties']
             required = prop['required'] if 'required' in prop else None

diff --git a/setup.py b/setup.py
@@ -12,7 +12,7 @@
 
 setup(
     name='gbqschema_converter',
-    version='1.1.0',
+    version='1.2.0',
     description="Library to convert Google BigQuery Table Schema into Json Schema",
     long_description=README,
     long_description_content_type="text/markdown",
@@ -21,7 +21,7 @@
     author_email="admin@dkisler.com",
     license='MIT',
     classifiers=[
-        "Development Status :: 3 - Alpha",
+        "Development Status :: 5 - Production/Stable",
         "Programming Language :: Python :: 3",
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",