new: Add support for loading JSON OpenAPI spec files (linode#629)

.github/workflows/docker-build.yml

@@ -17,4 +17,4 @@ jobs:
     - name: Build the Docker image
       run: docker build . --file Dockerfile --tag linode/cli:$(date +%s) --build-arg="github_token=$GITHUB_TOKEN"
       env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

Dockerfile

@@ -1,6 +1,7 @@
 FROM python:3.11-slim AS builder
 
 ARG linode_cli_version
+
 ARG github_token
 
 WORKDIR /src

Makefile

@@ -1,6 +1,8 @@
 #
 # Makefile for more convenient building of the Linode CLI and its baked content
 #
+
+# Test-related arguments
 MODULE :=
 TEST_CASE_COMMAND :=
 TEST_ARGS :=
@@ -9,7 +11,6 @@ ifdef TEST_CASE
 TEST_CASE_COMMAND = -k $(TEST_CASE)
 endif
 
-
 SPEC_VERSION ?= latest
 ifndef SPEC
 override SPEC = $(shell ./resolve_spec_url ${SPEC_VERSION})

linodecli/__init__.py

@@ -15,12 +15,7 @@
 from linodecli import plugins
 from linodecli.exit_codes import ExitCodes
 
-from .arg_helpers import (
-    bake_command,
-    register_args,
-    register_plugin,
-    remove_plugin,
-)
+from .arg_helpers import register_args, register_plugin, remove_plugin
 from .cli import CLI
 from .completion import get_completions
 from .configuration import ENV_TOKEN_NAME
@@ -103,7 +98,7 @@ def main():  # pylint: disable=too-many-branches,too-many-statements
         if parsed.action is None:
             print("No spec provided, cannot bake", file=sys.stderr)
             sys.exit(ExitCodes.ARGUMENT_ERROR)
-        bake_command(cli, parsed.action)
+        cli.bake(parsed.action)
         sys.exit(ExitCodes.SUCCESS)
     elif cli.ops is None:
         # if not spec was found and we weren't baking, we're doomed

linodecli/arg_helpers.py

@@ -2,16 +2,10 @@
 """
 Argument parser for the linode CLI
 """
-
-import os
 import sys
 from importlib import import_module
 
-import requests
-import yaml
-
 from linodecli import plugins
-from linodecli.exit_codes import ExitCodes
 from linodecli.helpers import (
     register_args_shared,
     register_debug_arg,
@@ -169,24 +163,3 @@ def remove_plugin(plugin_name, config):
 
     config.write_config()
     return f"Plugin {plugin_name} removed", 0
-
-
-def bake_command(cli, spec_loc):
-    """
-    Handle a bake command from args
-    """
-    try:
-        if os.path.exists(os.path.expanduser(spec_loc)):
-            with open(os.path.expanduser(spec_loc), encoding="utf-8") as f:
-                spec = yaml.safe_load(f.read())
-        else:  # try to GET it
-            resp = requests.get(spec_loc, timeout=120)
-            if resp.status_code == 200:
-                spec = yaml.safe_load(resp.content)
-            else:
-                raise RuntimeError(f"Request failed to {spec_loc}")
-    except Exception as e:
-        print(f"Could not load spec: {e}", file=sys.stderr)
-        sys.exit(ExitCodes.REQUEST_FAILED)
-
-    cli.bake(spec)

linodecli/baked/parsing.py

@@ -9,7 +9,7 @@
 
 # Sentence delimiter, split on a period followed by any type of
 # whitespace (space, new line, tab, etc.)
-REGEX_SENTENCE_DELIMITER = re.compile(r"\.(?:\s|$)")
+REGEX_SENTENCE_DELIMITER = re.compile(r"\W(?:\s|$)")
 
 # Matches on pattern __prefix__ at the beginning of a description
 # or after a comma

linodecli/cli.py

@@ -2,11 +2,17 @@
 Responsible for managing spec and routing commands to operations.
 """
 
+import contextlib
+import json
 import os
 import pickle
 import sys
+from json import JSONDecodeError
 from sys import version_info
+from typing import IO, Any, ContextManager, Dict
 
+import requests
+import yaml
 from openapi3 import OpenAPI
 
 from linodecli.api_request import do_request, get_all_pages
@@ -40,11 +46,19 @@ def __init__(self, version, base_url, skip_config=False):
         self.config = CLIConfig(self.base_url, skip_config=skip_config)
         self.load_baked()
 
-    def bake(self, spec):
+    def bake(self, spec_location: str):
         """
-        Generates ops and bakes them to a pickle
+        Generates ops and bakes them to a pickle.
+
+        :param spec_location: The URL or file path of the OpenAPI spec to parse.
         """
-        spec = OpenAPI(spec)
+
+        try:
+            spec = self._load_openapi_spec(spec_location)
+        except Exception as e:
+            print(f"Failed to load spec: {e}")
+            sys.exit(ExitCodes.REQUEST_FAILED)
+
         self.spec = spec
         self.ops = {}
         ext = {
@@ -206,3 +220,85 @@ def user_agent(self) -> str:
             f"linode-api-docs/{self.spec_version} "
             f"python/{version_info[0]}.{version_info[1]}.{version_info[2]}"
         )
+
+    @staticmethod
+    def _load_openapi_spec(spec_location: str) -> OpenAPI:
+        """
+        Attempts to load the raw OpenAPI spec (YAML or JSON) at the given location.
+
+        :param spec_location: The location of the OpenAPI spec.
+                              This can be a local path or a URL.
+
+        :returns: A tuple containing the loaded OpenAPI object and the parsed spec in
+                  dict format.
+        """
+
+        with CLI._get_spec_file_reader(spec_location) as f:
+            parsed = CLI._parse_spec_file(f)
+
+        return OpenAPI(parsed)
+
+    @staticmethod
+    @contextlib.contextmanager
+    def _get_spec_file_reader(
+        spec_location: str,
+    ) -> ContextManager[IO]:
+        """
+        Returns a reader for an OpenAPI spec file from the given location.
+
+        :param spec_location: The location of the OpenAPI spec.
+                      This can be a local path or a URL.
+
+        :returns: A context manager yielding the spec file's reader.
+        """
+
+        # Case for local file
+        local_path = os.path.expanduser(spec_location)
+        if os.path.exists(local_path):
+            f = open(local_path, "r", encoding="utf-8")
+
+            try:
+                yield f
+            finally:
+                f.close()
+
+            return
+
+        # Case for remote file
+        resp = requests.get(spec_location, stream=True, timeout=120)
+        if resp.status_code != 200:
+            raise RuntimeError(f"Failed to GET {spec_location}")
+
+        # We need to access the underlying urllib
+        # response here so we can return a reader
+        # usable in yaml.safe_load(...) and json.load(...)
+        resp.raw.decode_content = True
+
+        try:
+            yield resp.raw
+        finally:
+            resp.close()
+
+    @staticmethod
+    def _parse_spec_file(reader: IO) -> Dict[str, Any]:
+        """
+        Parses the given file reader into a dict and returns a dict.
+
+        :param reader: A reader for a YAML or JSON file.
+
+        :returns: The parsed file.
+        """
+
+        errors = []
+
+        try:
+            return yaml.safe_load(reader)
+        except yaml.YAMLError as err:
+            errors.append(str(err))
+
+        try:
+            return json.load(reader)
+        except JSONDecodeError as err:
+            errors.append(str(err))
+
+        raise ValueError(f"Failed to parse spec file: {'; '.join(errors)}")

tests/fixtures/cli_test_load.json

@@ -0,0 +1,95 @@
+{
+  "openapi": "3.0.1",
+  "info": {
+    "title": "API Specification",
+    "version": "1.0.0"
+  },
+  "servers": [
+    {
+      "url": "http://localhost/v4"
+    }
+  ],
+  "paths": {
+    "/foo/bar": {
+      "get": {
+        "summary": "get info",
+        "operationId": "fooBarGet",
+        "description": "This is description",
+        "responses": {
+          "200": {
+            "description": "Successful response",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "object",
+                  "properties": {
+                    "data": {
+                      "type": "array",
+                      "items": {
+                        "$ref": "#/components/schemas/OpenAPIResponseAttr"
+                      }
+                    },
+                    "page": {
+                      "$ref": "#/components/schemas/PaginationEnvelope/properties/page"
+                    },
+                    "pages": {
+                      "$ref": "#/components/schemas/PaginationEnvelope/properties/pages"
+                    },
+                    "results": {
+                      "$ref": "#/components/schemas/PaginationEnvelope/properties/results"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  },
+  "components": {
+    "schemas": {
+      "OpenAPIResponseAttr": {
+        "type": "object",
+        "properties": {
+          "filterable_result": {
+            "x-linode-filterable": true,
+            "type": "string",
+            "description": "Filterable result value"
+          },
+          "filterable_list_result": {
+            "x-linode-filterable": true,
+            "type": "array",
+            "items": {
+              "type": "string"
+            },
+            "description": "Filterable result value"
+          }
+        }
+      },
+      "PaginationEnvelope": {
+        "type": "object",
+        "properties": {
+          "pages": {
+            "type": "integer",
+            "readOnly": true,
+            "description": "The total number of pages.",
+            "example": 1
+          },
+          "page": {
+            "type": "integer",
+            "readOnly": true,
+            "description": "The current page.",
+            "example": 1
+          },
+          "results": {
+            "type": "integer",
+            "readOnly": true,
+            "description": "The total number of results.",
+            "example": 1
+          }
+        }
+      }
+    }
+  }
+}

tests/fixtures/cli_test_load.yaml

@@ -0,0 +1,64 @@
+openapi: 3.0.1
+info:
+  title: API Specification
+  version: 1.0.0
+servers:
+  - url: http://localhost/v4
+paths:
+  /foo/bar:
+    get:
+      summary: get info
+      operationId: fooBarGet
+      description: This is description
+      responses:
+        '200':
+          description: Successful response
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  data:
+                    type: array
+                    items:
+                      $ref: '#/components/schemas/OpenAPIResponseAttr'
+                  page:
+                    $ref: '#/components/schemas/PaginationEnvelope/properties/page'
+                  pages:
+                    $ref: '#/components/schemas/PaginationEnvelope/properties/pages'
+                  results:
+                    $ref: '#/components/schemas/PaginationEnvelope/properties/results'
+
+components:
+  schemas:
+    OpenAPIResponseAttr:
+      type: object
+      properties:
+        filterable_result:
+          x-linode-filterable: true
+          type: string
+          description: Filterable result value
+        filterable_list_result:
+          x-linode-filterable: true
+          type: array
+          items:
+            type: string
+          description: Filterable result value
+    PaginationEnvelope:
+      type: object
+      properties:
+        pages:
+          type: integer
+          readOnly: true
+          description: The total number of pages.
+          example: 1
+        page:
+          type: integer
+          readOnly: true
+          description: The current page.
+          example: 1
+        results:
+          type: integer
+          readOnly: true
+          description: The total number of results.
+          example: 1