feat(anta.cli): Add anta get tests (aristanetworks#843)

.pre-commit-config.yaml

@@ -108,3 +108,19 @@ repos:
           - --config=.github/markdownlint.yaml
           - --ignore-path=.github/markdownlintignore
           - --fix
+
+  - repo: local
+    hooks:
+      - id: examples-test
+        name: Generate examples/tests.yaml
+        entry: >-
+          sh -c "docs/scripts/generate_examples_tests.py"
+        language: python
+        types: [python]
+        files: anta/
+        verbose: true
+        pass_filenames: false
+        additional_dependencies:
+          - anta[cli]
+          # TODO: next can go once we have it added to anta properly
+          - numpydoc

anta/cli/get/__init__.py

@@ -17,3 +17,4 @@ def get() -> None:
 get.add_command(commands.from_ansible)
 get.add_command(commands.inventory)
 get.add_command(commands.tags)
+get.add_command(commands.tests)

anta/cli/get/commands.py

@@ -22,7 +22,7 @@
 from anta.cli.get.utils import inventory_output_options
 from anta.cli.utils import ExitCode, inventory_options
 
-from .utils import create_inventory_from_ansible, create_inventory_from_cvp, get_cv_token
+from .utils import create_inventory_from_ansible, create_inventory_from_cvp, explore_package, get_cv_token
 
 if TYPE_CHECKING:
     from anta.inventory import AntaInventory
@@ -132,3 +132,25 @@ def tags(inventory: AntaInventory, **kwargs: Any) -> None:
         tags.update(device.tags)
     console.print("Tags found:")
     console.print_json(json.dumps(sorted(tags), indent=2))
+
+
+@click.command
+@click.pass_context
+@click.option("--module", help="Filter tests by module name.", default="anta.tests", show_default=True)
+@click.option("--test", help="Filter by specific test name. If module is specified, searches only within that module.", type=str)
+@click.option("--short", help="Display test names without their inputs.", is_flag=True, default=False)
+@click.option("--count", help="Print only the number of tests found.", is_flag=True, default=False)
+def tests(ctx: click.Context, module: str, test: str | None, *, short: bool, count: bool) -> None:
+    """Show all builtin ANTA tests with an example output retrieved from each test documentation."""
+    try:
+        tests_found = explore_package(module, test_name=test, short=short, count=count)
+        if tests_found == 0:
+            console.print(f"""No test {f"'{test}' " if test else ""}found in '{module}'.""")
+        elif count:
+            if tests_found == 1:
+                console.print(f"There is 1 test available in '{module}'.")
+            else:
+                console.print(f"There are {tests_found} tests available in '{module}'.")
+    except ValueError as e:
+        logger.error(str(e))
+        ctx.exit(ExitCode.USAGE_ERROR)

anta/cli/get/utils.py

@@ -6,8 +6,14 @@
 from __future__ import annotations
 
 import functools
+import importlib
+import inspect
 import json
 import logging
+import pkgutil
+import re
+import sys
+import textwrap
 from pathlib import Path
 from sys import stdin
 from typing import Any, Callable
@@ -17,9 +23,11 @@
 import urllib3
 import yaml
 
+from anta.cli.console import console
 from anta.cli.utils import ExitCode
 from anta.inventory import AntaInventory
 from anta.inventory.models import AntaInventoryHost, AntaInventoryInput
+from anta.models import AntaTest
 
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
@@ -204,3 +212,148 @@ def create_inventory_from_ansible(inventory: Path, output: Path, ansible_group:
         raise ValueError(msg)
     ansible_hosts = deep_yaml_parsing(ansible_inventory)
     write_inventory_to_file(ansible_hosts, output)
+
+
+def explore_package(module_name: str, test_name: str | None = None, *, short: bool = False, count: bool = False) -> int:
+    """Parse ANTA test submodules recursively and print AntaTest examples.
+
+    Parameters
+    ----------
+    module_name
+        Name of the module to explore (e.g., 'anta.tests.routing.bgp').
+    test_name
+        If provided, only show tests starting with this name.
+    short
+        If True, only print test names without their inputs.
+    count
+        If True, only count the tests.
+
+    Returns
+    -------
+    int:
+        The number of tests found.
+    """
+    try:
+        module_spec = importlib.util.find_spec(module_name)
+    except ModuleNotFoundError:
+        # Relying on module_spec check below.
+        module_spec = None
+    except ImportError as e:
+        msg = "`anta get tests --module <module>` does not support relative imports"
+        raise ValueError(msg) from e
+
+    # Giving a second chance adding CWD to PYTHONPATH
+    if module_spec is None:
+        try:
+            logger.info("Could not find module `%s`, injecting CWD in PYTHONPATH and retrying...", module_name)
+            sys.path = [str(Path.cwd()), *sys.path]
+            module_spec = importlib.util.find_spec(module_name)
+        except ImportError:
+            module_spec = None
+
+    if module_spec is None or module_spec.origin is None:
+        msg = f"Module `{module_name}` was not found!"
+        raise ValueError(msg)
+
+    tests_found = 0
+    if module_spec.submodule_search_locations:
+        for _, sub_module_name, ispkg in pkgutil.walk_packages(module_spec.submodule_search_locations):
+            qname = f"{module_name}.{sub_module_name}"
+            if ispkg:
+                tests_found += explore_package(qname, test_name=test_name, short=short, count=count)
+                continue
+            tests_found += find_tests_examples(qname, test_name, short=short, count=count)
+
+    else:
+        tests_found += find_tests_examples(module_spec.name, test_name, short=short, count=count)
+
+    return tests_found
+
+
+def find_tests_examples(qname: str, test_name: str | None, *, short: bool = False, count: bool = False) -> int:
+    """Print tests from `qname`, filtered by `test_name` if provided.
+
+    Parameters
+    ----------
+    qname
+        Name of the module to explore (e.g., 'anta.tests.routing.bgp').
+    test_name
+        If provided, only show tests starting with this name.
+    short
+        If True, only print test names without their inputs.
+    count
+        If True, only count the tests.
+
+    Returns
+    -------
+    int:
+        The number of tests found.
+    """
+    try:
+        qname_module = importlib.import_module(qname)
+    except (AssertionError, ImportError) as e:
+        msg = f"Error when importing `{qname}` using importlib!"
+        raise ValueError(msg) from e
+
+    module_printed = False
+    tests_found = 0
+
+    for _name, obj in inspect.getmembers(qname_module):
+        # Only retrieves the subclasses of AntaTest
+        if not inspect.isclass(obj) or not issubclass(obj, AntaTest) or obj == AntaTest:
+            continue
+        if test_name and not obj.name.startswith(test_name):
+            continue
+        if not module_printed:
+            if not count:
+                console.print(f"{qname}:")
+            module_printed = True
+        tests_found += 1
+        if count:
+            continue
+        print_test(obj, short=short)
+
+    return tests_found
+
+
+def print_test(test: type[AntaTest], *, short: bool = False) -> None:
+    """Print a single test.
+
+    Parameters
+    ----------
+    test
+        the representation of the AntaTest as returned by inspect.getmembers
+    short
+        If True, only print test names without their inputs.
+    """
+    if not test.__doc__ or (example := extract_examples(test.__doc__)) is None:
+        msg = f"Test {test.name} in module {test.__module__} is missing an Example"
+        raise LookupError(msg)
+    # Picking up only the inputs in the examples
+    # Need to handle the fact that we nest the routing modules in Examples.
+    # This is a bit fragile.
+    inputs = example.split("\n")
+    try:
+        test_name_line = next((i for i, input_entry in enumerate(inputs) if test.name in input_entry))
+    except StopIteration as e:
+        msg = f"Could not find the name of the test '{test.name}' in the Example section in the docstring."
+        raise ValueError(msg) from e
+    # TODO: handle not found
+    console.print(f"  {inputs[test_name_line].strip()}")
+    # Injecting the description
+    console.print(f"      # {test.description}", soft_wrap=True)
+    if not short and len(inputs) > test_name_line + 2:  # There are params
+        console.print(textwrap.indent(textwrap.dedent("\n".join(inputs[test_name_line + 1 : -1])), " " * 6))
+
+
+def extract_examples(docstring: str) -> str | None:
+    """Extract the content of the Example section in a Numpy docstring.
+
+    Returns
+    -------
+    str | None
+        The content of the section if present, None if the section is absent or empty.
+    """
+    pattern = r"Examples\s*--------\s*(.*)(?:\n\s*\n|\Z)"
+    match = re.search(pattern, docstring, flags=re.DOTALL)
+    return match[1].strip() if match and match[1].strip() != "" else None

anta/tests/aaa.py

@@ -167,14 +167,14 @@ class VerifyAuthenMethods(AntaTest):
     ```yaml
     anta.tests.aaa:
       - VerifyAuthenMethods:
-        methods:
-          - local
-          - none
-          - logging
-        types:
-          - login
-          - enable
-          - dot1x
+          methods:
+            - local
+            - none
+            - logging
+          types:
+            - login
+            - enable
+            - dot1x
     ```
     """
 

anta/tests/configuration.py

@@ -94,9 +94,9 @@ class VerifyRunningConfigLines(AntaTest):
     ```yaml
     anta.tests.configuration:
       - VerifyRunningConfigLines:
-            regex_patterns:
-                - "^enable password.*$"
-                - "bla bla"
+          regex_patterns:
+            - "^enable password.*$"
+            - "bla bla"
     ```
     """
 

anta/tests/flow_tracking.py

@@ -86,7 +86,7 @@ class VerifyHardwareFlowTrackerStatus(AntaTest):
     --------
     ```yaml
     anta.tests.flow_tracking:
-      - VerifyFlowTrackingHardware:
+      - VerifyHardwareFlowTrackerStatus:
           trackers:
             - name: FLOW-TRACKER
               record_export:

anta/tests/greent.py

@@ -25,8 +25,9 @@ class VerifyGreenTCounters(AntaTest):
     --------
     ```yaml
     anta.tests.greent:
-      - VerifyGreenT:
+      - VerifyGreenTCounters:
     ```
+
     """
 
     description = "Verifies if the GreenT counters are incremented."
@@ -56,8 +57,9 @@ class VerifyGreenT(AntaTest):
     --------
     ```yaml
     anta.tests.greent:
-      - VerifyGreenTCounters:
+      - VerifyGreenT:
     ```
+
     """
 
     description = "Verifies if a GreenT policy other than the default is created."

anta/tests/routing/isis.py

@@ -349,7 +349,6 @@ class VerifyISISSegmentRoutingAdjacencySegments(AntaTest):
                   - interface: Ethernet2
                     address: 10.0.1.3
                     sid_origin: dynamic
-
     ```
     """
 
@@ -530,21 +529,21 @@ class VerifyISISSegmentRoutingTunnels(AntaTest):
     --------
     ```yaml
     anta.tests.routing:
-    isis:
+      isis:
         - VerifyISISSegmentRoutingTunnels:
             entries:
-            # Check only endpoint
-            - endpoint: 1.0.0.122/32
-            # Check endpoint and via TI-LFA
-            - endpoint: 1.0.0.13/32
-              vias:
-                - type: tunnel
-                  tunnel_id: ti-lfa
-            # Check endpoint and via IP routers
-            - endpoint: 1.0.0.14/32
-              vias:
-                - type: ip
-                  nexthop: 1.1.1.1
+              # Check only endpoint
+              - endpoint: 1.0.0.122/32
+              # Check endpoint and via TI-LFA
+              - endpoint: 1.0.0.13/32
+                vias:
+                  - type: tunnel
+                    tunnel_id: ti-lfa
+              # Check endpoint and via IP routers
+              - endpoint: 1.0.0.14/32
+                vias:
+                  - type: ip
+                    nexthop: 1.1.1.1
     ```
     """
 

anta/tests/security.py

@@ -483,10 +483,10 @@ class VerifyBannerLogin(AntaTest):
     ```yaml
     anta.tests.security:
       - VerifyBannerLogin:
-            login_banner: |
-                # Copyright (c) 2023-2024 Arista Networks, Inc.
-                # Use of this source code is governed by the Apache License 2.0
-                # that can be found in the LICENSE file.
+          login_banner: |
+            # Copyright (c) 2023-2024 Arista Networks, Inc.
+            # Use of this source code is governed by the Apache License 2.0
+            # that can be found in the LICENSE file.
     ```
     """
 
@@ -525,10 +525,10 @@ class VerifyBannerMotd(AntaTest):
     ```yaml
     anta.tests.security:
       - VerifyBannerMotd:
-            motd_banner: |
-                # Copyright (c) 2023-2024 Arista Networks, Inc.
-                # Use of this source code is governed by the Apache License 2.0
-                # that can be found in the LICENSE file.
+          motd_banner: |
+            # Copyright (c) 2023-2024 Arista Networks, Inc.
+            # Use of this source code is governed by the Apache License 2.0
+            # that can be found in the LICENSE file.
     ```
     """
 

anta/tests/system.py

@@ -107,15 +107,15 @@ class VerifyCoredump(AntaTest):
     * Success: The test will pass if there are NO core dump(s) in /var/core.
     * Failure: The test will fail if there are core dump(s) in /var/core.
 
-    Info
-    ----
+    Notes
+    -----
     * This test will NOT check for minidump(s) generated by certain agents in /var/core/minidump.
 
     Examples
     --------
     ```yaml
     anta.tests.system:
-      - VerifyCoreDump:
+      - VerifyCoredump:
     ```
     """