From 6ba475c8945f083958e2c6cbbbadb5996ff05710 Mon Sep 17 00:00:00 2001 From: "Michael \"M3\" Lasevich" Date: Fri, 24 Jan 2025 17:32:43 -0600 Subject: [PATCH 1/8] Porting customize/filters/tests functionality from j2cli --- src/jinjanator/cli.py | 14 +++- src/jinjanator/customize.py | 150 ++++++++++++++++++++++++++++++++++++ 2 files changed, 163 insertions(+), 1 deletion(-) create mode 100644 src/jinjanator/customize.py diff --git a/src/jinjanator/cli.py b/src/jinjanator/cli.py index 22c93eb..cb96e2f 100644 --- a/src/jinjanator/cli.py +++ b/src/jinjanator/cli.py @@ -20,6 +20,7 @@ from . import filters, formats, version from .context import read_context_data +from . import customize class FilePathLoader(jinja2.BaseLoader): @@ -220,6 +221,9 @@ def parse_args( help="Suppress informational messages", ) + # add args for customize support + customize.add_args(parser) + parser.add_argument( "-o", "--output-file", @@ -342,13 +346,21 @@ def render_command( args.import_env, ) + customizations = customize.from_file(args.customize) + + context = customizations.alter_context(context) + renderer = Jinja2TemplateRenderer( cwd, args.undefined, - j2_env_params={}, + j2_env_params=customizations.j2_environment_params(), plugin_hook_callers=plugin_hook_callers, ) + customize.apply(customizations, renderer._env, + filters=args.filters, + tests=args.tests) + try: result = renderer.render(args.template, context) except jinja2.exceptions.UndefinedError as e: diff --git a/src/jinjanator/customize.py b/src/jinjanator/customize.py new file mode 100644 index 0000000..c8c0158 --- /dev/null +++ b/src/jinjanator/customize.py @@ -0,0 +1,150 @@ +""" +Add Customize/Filters/Tests functionality from J2CLI + +This code was ported from https://github.com/kolypto/j2cli + +""" +import inspect +import types +from argparse import ArgumentParser +from importlib.machinery import SourceFileLoader +from typing import List, Dict + +import jinja2 + + +def imp_load_source(module_name, module_path): + """ + Drop-in Replacement for imp.load_source() function in pre-3.12 python + + Source: https://github.com/python/cpython/issues/104212 + """ + loader = SourceFileLoader(module_name, module_path) + module = types.ModuleType(loader.name) + loader.exec_module(module) + return module + + +class CustomizationModule(object): + """ The interface for customization functions, defined as module-level + functions """ + + def __init__(self, module=None): + if module is not None: + def howcall(*args): + print(args) + exit(1) + + # Import every module function as a method on ourselves + for name in self._IMPORTED_METHOD_NAMES: + try: + setattr(self, name, getattr(module, name)) + except AttributeError: + pass + + # stubs + + def j2_environment_params(self): + return {} + + def j2_environment(self, env): + return env + + def alter_context(self, context): + return context + + def extra_filters(self): + return {} + + def extra_tests(self): + return {} + + _IMPORTED_METHOD_NAMES = [ + f.__name__ + for f in ( + j2_environment_params, j2_environment, alter_context, extra_filters, + extra_tests)] + + +def from_file(filename: str) -> CustomizationModule: + """ Create Customize object """ + if filename is not None: + return CustomizationModule( + imp_load_source('customize-module', filename) + ) + return CustomizationModule(None) + + +def import_functions(filename: str): + """ Import functions from file, return as a dictionary """ + m = imp_load_source('imported-funcs', filename) + return dict((name, func) + for name, func in inspect.getmembers(m) + if inspect.isfunction(func)) + + +def register_filters(renderer_env: jinja2.Environment, filters: Dict): + """ Register additional filters """ + renderer_env.filters.update(filters) + + +def register_tests(renderer_env: jinja2.Environment, tests: Dict): + """ Register additional tests """ + renderer_env.tests.update(tests) + + +def import_filters(renderer_env: jinja2.Environment, filename: str): + """ Import filters from a file """ + register_filters(renderer_env, import_functions(filename)) + + +def import_tests(renderer_env: jinja2.Environment, filename: str): + """ Import tests from a file """ + register_tests(renderer_env, import_functions(filename)) + + +def apply(customize: CustomizationModule, + renderer_env: jinja2.Environment, + filters: List[str], + tests: List[str]): + """ Apply customizations """ + customize.j2_environment(renderer_env) + + for fname in filters: + import_filters(renderer_env, fname) + + for fname in tests: + import_tests(renderer_env, fname) + + register_filters(renderer_env, customize.extra_filters()) + + register_tests(renderer_env, customize.extra_tests()) + + +def add_args(parser: ArgumentParser) -> ArgumentParser: + """ Add args to the parser """ + parser.add_argument( + '--customize', + default=None, + metavar='python-file.py', + dest='customize', + help='A Python file that implements hooks to ' + 'fine-tune the jinjanator behavior') + + parser.add_argument( + '--filters', + nargs='+', + default=[], + metavar='python-file', + dest='filters', + help='Load custom Jinja2 filters from a Python file: ' + 'all top-level functions are imported.') + + parser.add_argument( + '--tests', + nargs='+', + default=[], + metavar='python-file', + dest='tests', + help='Load custom Jinja2 tests from a Python file.') + return parser From 07559b9cd708c55316c833c5190057ccfbf018e3 Mon Sep 17 00:00:00 2001 From: "Michael \"M3\" Lasevich" Date: Mon, 27 Jan 2025 09:06:13 -0600 Subject: [PATCH 2/8] code cleanup/refactor --- src/jinjanator/cli.py | 22 +++--- src/jinjanator/customize.py | 139 +++++++++++++++++------------------- 2 files changed, 77 insertions(+), 84 deletions(-) diff --git a/src/jinjanator/cli.py b/src/jinjanator/cli.py index cb96e2f..2f93acd 100644 --- a/src/jinjanator/cli.py +++ b/src/jinjanator/cli.py @@ -18,9 +18,9 @@ import jinjanator_plugins import pluggy -from . import filters, formats, version +from . import customize, filters, formats, version from .context import read_context_data -from . import customize +from .customize import CustomizationModule class FilePathLoader(jinja2.BaseLoader): @@ -72,23 +72,23 @@ def __init__( j2_env_params.setdefault("extensions", self.ENABLED_EXTENSIONS) j2_env_params.setdefault("loader", FilePathLoader(cwd)) - self._env = jinja2.Environment(**j2_env_params, autoescape=False) # noqa: S701 + self.env = jinja2.Environment(**j2_env_params, autoescape=False) # noqa: S701 for plugin_globals in plugin_hook_callers.plugin_globals(): - self._env.globals |= plugin_globals + self.env.globals |= plugin_globals for plugin_filters in plugin_hook_callers.plugin_filters(): - self._env.filters |= plugin_filters + self.env.filters |= plugin_filters for plugin_tests in plugin_hook_callers.plugin_tests(): - self._env.tests |= plugin_tests + self.env.tests |= plugin_tests for plugin_extensions in plugin_hook_callers.plugin_extensions(): for extension in plugin_extensions: - self._env.add_extension(extension) + self.env.add_extension(extension) def render(self, template_name: str, context: Mapping[str, str]) -> str: - return self._env.get_template(template_name).render(context) + return self.env.get_template(template_name).render(context) class UniqueStore(argparse.Action): @@ -346,7 +346,7 @@ def render_command( args.import_env, ) - customizations = customize.from_file(args.customize) + customizations = CustomizationModule.from_file(args.customize) context = customizations.alter_context(context) @@ -357,9 +357,7 @@ def render_command( plugin_hook_callers=plugin_hook_callers, ) - customize.apply(customizations, renderer._env, - filters=args.filters, - tests=args.tests) + customize.apply(customizations, renderer.env, filters=args.filters, tests=args.tests) try: result = renderer.render(args.template, context) diff --git a/src/jinjanator/customize.py b/src/jinjanator/customize.py index c8c0158..58c1c0e 100644 --- a/src/jinjanator/customize.py +++ b/src/jinjanator/customize.py @@ -2,112 +2,106 @@ Add Customize/Filters/Tests functionality from J2CLI This code was ported from https://github.com/kolypto/j2cli - """ + +import contextlib import inspect -import types + from argparse import ArgumentParser +from collections.abc import Mapping from importlib.machinery import SourceFileLoader -from typing import List, Dict +from types import FunctionType, ModuleType +from typing import Any, ClassVar, Optional import jinja2 -def imp_load_source(module_name, module_path): +def imp_load_source(module_name: str, module_path: str) -> ModuleType: """ Drop-in Replacement for imp.load_source() function in pre-3.12 python Source: https://github.com/python/cpython/issues/104212 """ loader = SourceFileLoader(module_name, module_path) - module = types.ModuleType(loader.name) + module = ModuleType(loader.name) loader.exec_module(module) return module -class CustomizationModule(object): - """ The interface for customization functions, defined as module-level - functions """ +class CustomizationModule: + """The interface for customization functions, defined as module-level + functions""" - def __init__(self, module=None): + def __init__(self, module: Optional[ModuleType] = None): if module is not None: - def howcall(*args): - print(args) - exit(1) - # Import every module function as a method on ourselves - for name in self._IMPORTED_METHOD_NAMES: - try: + with contextlib.suppress(AttributeError): + for name in self._IMPORTED_METHOD_NAMES: setattr(self, name, getattr(module, name)) - except AttributeError: - pass # stubs - def j2_environment_params(self): + def j2_environment_params(self) -> dict[str, Any]: return {} - def j2_environment(self, env): + def j2_environment(self, env: jinja2.Environment) -> jinja2.Environment: return env - def alter_context(self, context): + def alter_context(self, context: Mapping[str, Any]) -> Mapping[str, Any]: return context - def extra_filters(self): + def extra_filters(self) -> Mapping[str, FunctionType]: return {} - def extra_tests(self): + def extra_tests(self) -> Mapping[str, FunctionType]: return {} - _IMPORTED_METHOD_NAMES = [ + _IMPORTED_METHOD_NAMES: ClassVar = [ f.__name__ - for f in ( - j2_environment_params, j2_environment, alter_context, extra_filters, - extra_tests)] - + for f in (j2_environment_params, j2_environment, alter_context, extra_filters, extra_tests) + ] -def from_file(filename: str) -> CustomizationModule: - """ Create Customize object """ - if filename is not None: - return CustomizationModule( - imp_load_source('customize-module', filename) - ) - return CustomizationModule(None) + @classmethod + def from_file(cls, filename: str) -> "CustomizationModule": + """Create Customize object""" + if filename is not None: + return cls(imp_load_source("customize-module", filename)) + return cls(None) -def import_functions(filename: str): - """ Import functions from file, return as a dictionary """ - m = imp_load_source('imported-funcs', filename) - return dict((name, func) - for name, func in inspect.getmembers(m) - if inspect.isfunction(func)) +def import_functions(filename: str) -> Mapping[str, FunctionType]: + """Import functions from file, return as a dictionary""" + m = imp_load_source("imported-funcs", filename) + return {name: func for name, func in inspect.getmembers(m) if inspect.isfunction(func)} -def register_filters(renderer_env: jinja2.Environment, filters: Dict): - """ Register additional filters """ - renderer_env.filters.update(filters) +def register_filters(j2env: jinja2.Environment, filters: Mapping[str, FunctionType]) -> None: + """Register additional filters""" + j2env.filters.update(filters) -def register_tests(renderer_env: jinja2.Environment, tests: Dict): - """ Register additional tests """ - renderer_env.tests.update(tests) +def register_tests(j2env: jinja2.Environment, tests: Mapping[str, FunctionType]) -> None: + """Register additional tests""" + j2env.tests.update(tests) # type: ignore[arg-type] -def import_filters(renderer_env: jinja2.Environment, filename: str): - """ Import filters from a file """ +def import_filters(renderer_env: jinja2.Environment, filename: str) -> None: + """Import filters from a file""" register_filters(renderer_env, import_functions(filename)) -def import_tests(renderer_env: jinja2.Environment, filename: str): - """ Import tests from a file """ +def import_tests(renderer_env: jinja2.Environment, filename: str) -> None: + """Import tests from a file""" register_tests(renderer_env, import_functions(filename)) -def apply(customize: CustomizationModule, - renderer_env: jinja2.Environment, - filters: List[str], - tests: List[str]): - """ Apply customizations """ +def apply( + customize: CustomizationModule, + renderer_env: jinja2.Environment, + filters: list[str], + tests: list[str], +) -> None: + """Apply customizations""" customize.j2_environment(renderer_env) for fname in filters: @@ -122,29 +116,30 @@ def apply(customize: CustomizationModule, def add_args(parser: ArgumentParser) -> ArgumentParser: - """ Add args to the parser """ + """Add args to the parser""" parser.add_argument( - '--customize', + "--customize", default=None, - metavar='python-file.py', - dest='customize', - help='A Python file that implements hooks to ' - 'fine-tune the jinjanator behavior') + metavar="python-file.py", + dest="customize", + help="A Python file that implements hooks to fine-tune the jinjanator behavior", + ) parser.add_argument( - '--filters', - nargs='+', + "--filters", + nargs="+", default=[], - metavar='python-file', - dest='filters', - help='Load custom Jinja2 filters from a Python file: ' - 'all top-level functions are imported.') + metavar="python-file", + dest="filters", + help="Load custom Jinja2 filters from a Python file: all top-level functions are imported.", + ) parser.add_argument( - '--tests', - nargs='+', + "--tests", + nargs="+", default=[], - metavar='python-file', - dest='tests', - help='Load custom Jinja2 tests from a Python file.') + metavar="python-file", + dest="tests", + help="Load custom Jinja2 tests from a Python file.", + ) return parser From 7c62a8280507708f4bd034d7ebe9196cdc89aa97 Mon Sep 17 00:00:00 2001 From: "Michael \"M3\" Lasevich" Date: Mon, 27 Jan 2025 12:17:07 -0600 Subject: [PATCH 3/8] refactor code to reduce line count --- src/jinjanator/cli.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/jinjanator/cli.py b/src/jinjanator/cli.py index 2f93acd..9f4f428 100644 --- a/src/jinjanator/cli.py +++ b/src/jinjanator/cli.py @@ -308,7 +308,7 @@ def render_command( # We always expect a file; # unless the user wants 'env', and there's no input file provided. - if args.format == "env": + if args.format == "env" and args.data is None: """ With the "env" format, if no dotenv filename is provided, we have two options: 1. The user wants to use the current @@ -325,12 +325,7 @@ def render_command( And this is what we're going to do here as well. The script, however, would give the user a hint that they should use '-'. """ - if str(args.data) == "-": - input_data_f = stdin - elif args.data is None: - input_data_f = None - else: - input_data_f = args.data.open() + input_data_f = None else: input_data_f = stdin if args.data is None or str(args.data) == "-" else args.data.open() From 3606f56989c04dc93fc06841dfa8e5638b16ef51 Mon Sep 17 00:00:00 2001 From: "Michael \"M3\" Lasevich" Date: Mon, 27 Jan 2025 15:21:38 -0600 Subject: [PATCH 4/8] Update readme --- README.md | 171 ++++++++++++++++++++++++++++++++++++ src/jinjanator/customize.py | 11 ++- 2 files changed, 180 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 32f2387..fb28e0b 100644 --- a/README.md +++ b/README.md @@ -155,6 +155,17 @@ Options: error will be raised). * `--version`: prints the version of the tool and the Jinja2 package installed. +Customization Options: + +These options were ported from the j2cli tool for backwards compatibility (See customization section below) + +* `--filters PYTHON_FILE [FILE2] ...` - specify a python file containing additional filters as simple functions. You can specify more than one file at a time. NOTE: if this is the last argument before specifying the template filename, you will need to add a `--` separator before the template filename. + +* `--tests PYTHON_FILE [FILE2] ...` - specify a python file containing additional tests as simple functions. You can specify more than one file at a time. NOTE: if this is the last argument before specifying the template filename, you will need to add a `--` separator before the template filename. + +* `--customize PYTHON_FILE` - specify a customization python file. This file can modify context, add filters/tests or change J2 configuration. + + There is some special behavior with environment variables: * When `data` is not provided (data is `-`), `--format` defaults to @@ -352,6 +363,166 @@ Notice that there must be quotes around the environment variable name when it is a literal string. +## Customization + +(this functionality was ported from j2cli) + +Jinjanator now supports customizing your Jinja2 template processing via two methods - via simple files containing custom filters or tests, or via a more advanced "customize" file that allows you to do all of the above as well as modify core configuration of the Jinja2 engine + +### Via filters/tests files + +The simplest way to add additional filters or tests is via a "filters" or "tests" files. These files are simple python files with function. Each function becomes a filter or test. Examples: + +`filters.py` + +```python +# Simple filters file + +def parentheses(message): + """ Put message in parenthesis """ + return f"({message})" + +``` + +`tests.py` + +```python +# Example of simple tests file + +def an_odd_number(number): + """ test if number is odd """ + return True if (number % 2) else False +``` + +And a template that uses them: +``` +{% for x in range(4) %} +{{x}} is: {% if x is an_odd_number %} + {{- "odd" | parentheses }} + {%- else %} + {{- "even" | parentheses }} + {%- endif %} +{%- endfor %} +``` + +The output is: + +``` +$ jinjanate --filter ./filters.py --test ./tests.py -- simple.j2 + +0 is: (even) +1 is: (odd) +2 is: (even) +3 is: (odd) + +``` + +You can include multiple functions in each file and/or use multiple files as needed. + +### Using via a Customize File + +A more advanced way to customize your template processing is via a "customize" file. + +Customize file allows you to: + +* Pass additional keywords to Jinja2 environment +* Modify the context before it's used for rendering +* Register custom filters and tests + +This is done through *hooks* that you implement in a customization file in Python language. Each hook is a plain functions at the module level with the exact name as shown below. + +The following hooks are available: + +* `j2_environment_params() -> dict`: returns a `dict` of additional parameters for + [Jinja2 Environment](http://jinja.pocoo.org/docs/2.10/api/#jinja2.Environment). +* `j2_environment(env: Environment) -> Environment`: lets you customize the `Environment` object. +* `alter_context(context: dict) -> dict`: lets you modify the context variables that are going to be + used for template rendering. You can do all sorts of pre-processing here. +* `extra_filters() -> dict`: returns a `dict` with extra filters for Jinja2 +* `extra_tests() -> dict`: returns a `dict` with extra tests for Jinja2 + +All of them are optional. + +The example customization.py file for your reference: + +```python +# +# Example customization.py file for jinjanator +# Contains hooks that modify the way jinjanator is initialized and used + + +def j2_environment_params(): + """ Extra parameters for the Jinja2 Environment """ + # Jinja2 Environment configuration + # http://jinja.pocoo.org/docs/2.10/api/#jinja2.Environment + return dict( + # Just some examples + + # Change block start/end strings + block_start_string='<%', + block_end_string='%>', + # Change variable strings + variable_start_string='<<', + variable_end_string='>>', + # Remove whitespace around blocks + trim_blocks=True, + lstrip_blocks=True, + # Enable line statements: + # http://jinja.pocoo.org/docs/2.10/templates/#line-statements + line_statement_prefix='#', + # Keep \n at the end of a file + keep_trailing_newline=True, + # Enable custom extensions + # http://jinja.pocoo.org/docs/2.10/extensions/#jinja-extensions + extensions=('jinja2.ext.i18n',), + ) + + +def j2_environment(env): + """ Modify Jinja2 environment + + :param env: jinja2.environment.Environment + :rtype: jinja2.environment.Environment + """ + env.globals.update( + my_function=lambda v: 'my function says "{}"'.format(v) + ) + return env + + +def alter_context(context): + """ Modify the context and return it """ + # An extra variable + context['ADD'] = '127' + return context + + +def extra_filters(): + """ Declare some custom filters. + + Returns: dict(name = function) + """ + return dict( + # Example: {{ var | parentheses }} + parentheses=lambda t: '(' + t + ')', + ) + + +def extra_tests(): + """ Declare some custom tests + + Returns: dict(name = function) + """ + return dict( + # Example: {% if a|int is custom_odd %}odd{% endif %} + custom_odd=lambda n: True if (n % 2) else False + ) + +# + +``` +You can only have one customize file per run. + ## Chat If you'd like to chat with the jinjanator community, join us on diff --git a/src/jinjanator/customize.py b/src/jinjanator/customize.py index 58c1c0e..3fc6949 100644 --- a/src/jinjanator/customize.py +++ b/src/jinjanator/customize.py @@ -117,6 +117,12 @@ def apply( def add_args(parser: ArgumentParser) -> ArgumentParser: """Add args to the parser""" + vargs_warning = ( + "NOTE: Due to the way python implements options, if this is" + " the last option before the template filename, you need" + " to add a -- separator before the template filename" + ) + parser.add_argument( "--customize", default=None, @@ -131,7 +137,8 @@ def add_args(parser: ArgumentParser) -> ArgumentParser: default=[], metavar="python-file", dest="filters", - help="Load custom Jinja2 filters from a Python file: all top-level functions are imported.", + help="Load custom Jinja2 filters from a Python file(s). " + f"All top-level functions are imported as filters. {vargs_warning}", ) parser.add_argument( @@ -140,6 +147,6 @@ def add_args(parser: ArgumentParser) -> ArgumentParser: default=[], metavar="python-file", dest="tests", - help="Load custom Jinja2 tests from a Python file.", + help=f"Load custom Jinja2 tests from a Python file(s). {vargs_warning}", ) return parser From cdae3b5b4aa933340485a0fc8e9ff1752498df00 Mon Sep 17 00:00:00 2001 From: "Michael \"M3\" Lasevich" Date: Mon, 27 Jan 2025 17:44:41 -0600 Subject: [PATCH 5/8] Add tests --- tests/test_argparse.py | 5 + tests/test_customize.py | 227 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 232 insertions(+) create mode 100644 tests/test_customize.py diff --git a/tests/test_argparse.py b/tests/test_argparse.py index 480ed16..ca03751 100644 --- a/tests/test_argparse.py +++ b/tests/test_argparse.py @@ -41,6 +41,11 @@ def test_invalid_arg() -> None: ["-e", "env"], ["-f", "env"], ["-o", "output"], + ["--customize", "customize.py"], + ["--filters", "filename.py", "--"], + ["--filters", "filename.py", "filename2.py", "--"], + ["--tests", "filename.py", "--"], + ["--tests", "filename.py", "filename2.py", "--"], ], ) def test_args(args: list[str]) -> None: diff --git a/tests/test_customize.py b/tests/test_customize.py new file mode 100644 index 0000000..449d4d6 --- /dev/null +++ b/tests/test_customize.py @@ -0,0 +1,227 @@ +from argparse import Namespace +from dataclasses import dataclass +from pathlib import Path +from textwrap import dedent +from typing import Callable + +import pytest + +from jinjanator.cli import render_command + + +DirMakerTool = Callable[..., Namespace] + + +@dataclass +class FileContent: + """Holds filename and content""" + + filename: str + content: str + dedent: bool = True + + @property + def clean_content(self) -> str: + """Get cleaned up content""" + return dedent(self.content) if self.dedent else self.content + + +@pytest.fixture +def dir_maker(tmp_path: Path) -> DirMakerTool: + """Maker""" + + def _dir_maker(**kwargs: FileContent) -> Namespace: + result = {} + for name, file in kwargs.items(): + filename = tmp_path / f"{file.filename}" + filename.write_text(file.clean_content) + result[name] = str(filename) + return Namespace(**result) + + return _dir_maker + + +def test_two_custom_filters(dir_maker: DirMakerTool) -> None: + files = dir_maker( + template=FileContent( + "template.j2", + """ + {{- "Hello, World!" | with_parens | reverse -}} + """, + ), + data=FileContent("data.env", "key=value"), + filter=FileContent( + "filter.py", + """ + + def with_parens(message): + return f"({message})" + + """, + ), + filter2=FileContent( + "filter2.py", + """ + + def reverse(message): + return message[::=1] + + """, + ), + ) + + assert ")!dlroW ,olleH(" == render_command( + Path.cwd(), + {}, + None, + ["", "--filters", files.filter, "--", files.template, files.data], + ) + + +@pytest.mark.parametrize(("key", "expected"), [("Hello, World!", "(Hello, World!)")]) +def test_custom_filter(dir_maker: DirMakerTool, key: str, expected: str) -> None: + files = dir_maker( + template=FileContent( + "template.j2", + """ + {{- key | with_parens -}} + """, + ), + data=FileContent( + "data.env", + f""" + key="{key}" + """, + ), + filter=FileContent( + "filter.py", + """ + + def with_parens(message): + return f"({message})" + + """, + ), + ) + + assert expected == render_command( + Path.cwd(), + {}, + None, + ["", "--filter", files.filter, "--", files.template, files.data], + ) + + +@pytest.mark.parametrize( + ("key", "expected"), + [ + ("value", "NO"), + ("(value)", "YES"), + ], +) +def test_custom_test(dir_maker: DirMakerTool, key: str, expected: str) -> None: + files = dir_maker( + template=FileContent( + "template.j2", + """ + {%- if key is in_parens -%} + YES + {%- else -%} + NO + {%- endif -%} + """, + ), + data=FileContent( + "data.env", + f""" + key={key} + """, + ), + test=FileContent( + "test.py", + """ + + def in_parens(message): + return message and message[0]=="(" and message[-1] == ")" + + """, + ), + ) + + assert expected == render_command( + Path.cwd(), + {}, + None, + ["", "--tests", files.test, "--", files.template, files.data], + ) + + +@pytest.mark.parametrize(("key", "expected"), [("something", "YES"), ("(something)", "YES")]) +def test_custom_filter_and_test(dir_maker: DirMakerTool, key: str, expected: str) -> None: + files = dir_maker( + template=FileContent( + "template.j2", + """ + {%- if key | with_parens is in_parens -%} + YES + {%- else %} + NO + {%- endif -%} + """, + ), + data=FileContent("data.env", f"key={key}"), + filter=FileContent( + "filter.py", + """ + + def with_parens(message): + return f"({message})" + + """, + ), + test=FileContent( + "test.py", + """ + + def in_parens(message): + return message and message[0]=="(" and message[-1] == ")" + + """, + ), + ) + + assert expected == render_command( + Path.cwd(), + {}, + None, + ["", "--filters", files.filter, "--tests", files.test, "--", files.template, files.data], + ) + + +def test_customize_file(dir_maker: DirMakerTool) -> None: + files = dir_maker( + template=FileContent("template.j2", "<<- key >> works, {{ key }} doesn't"), + data=FileContent("data.env", "key=value"), + customize=FileContent( + "customize.py", + """ + + def j2_environment_params(): + return dict( + # Change block start/end strings + block_start_string='<%', + block_end_string='%>', + # Change variable strings + variable_start_string='<<', + variable_end_string='>>') + + """, + ), + ) + + assert "value works, {{ key }} doesn't" == render_command( + Path.cwd(), + {}, + None, + ["", "--customize", files.customize, files.template, files.data], + ) From c3ebb0b0a79304fcf97bcf48cebde3c207e4cf69 Mon Sep 17 00:00:00 2001 From: "Michael \"M3\" Lasevich" Date: Wed, 29 Jan 2025 09:44:29 -0600 Subject: [PATCH 6/8] Fix --filters/--tests argument behavior when providing multiple files --- README.md | 9 +++++---- src/jinjanator/customize.py | 22 ++++++++++------------ tests/test_customize.py | 24 ++++++++++++------------ 3 files changed, 27 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index fb28e0b..5ded87c 100644 --- a/README.md +++ b/README.md @@ -159,12 +159,13 @@ Customization Options: These options were ported from the j2cli tool for backwards compatibility (See customization section below) -* `--filters PYTHON_FILE [FILE2] ...` - specify a python file containing additional filters as simple functions. You can specify more than one file at a time. NOTE: if this is the last argument before specifying the template filename, you will need to add a `--` separator before the template filename. +* `--filters PYTHON_FILE` - specify a python file containing additional j2 filters as simple functions. You can use this option more than once to include multiple files. + * NOTE: while this option's behavior matches j2cli documentation, but does not match j2cli implementation. If you are migrating from j2cli and use more than one file, you will need to adjust your cli args from `... --filters file1.py file2.py ...` to `... --filters file1.py --filters file2.py ...``. -* `--tests PYTHON_FILE [FILE2] ...` - specify a python file containing additional tests as simple functions. You can specify more than one file at a time. NOTE: if this is the last argument before specifying the template filename, you will need to add a `--` separator before the template filename. - -* `--customize PYTHON_FILE` - specify a customization python file. This file can modify context, add filters/tests or change J2 configuration. +* `--tests PYTHON_FILE [FILE2] ...` - specify a python file containing additional j2 tests as simple functions. You can use this option more than once to include multiple files. + * NOTE: while this option's behavior matches j2cli documentation, but does not match j2cli implementation. If you are migrating from j2cli and use more than one file, you will need to adjust your cli args from `... --tests file1.py file2.py ...` to `... --tests file1.py --tests file2.py ...``. +* `--customize PYTHON_FILE` - specify a customization python file. This file can modify context, add filters/tests or change J2 configuration. Unlike `--filters` or `--tests` - this option can only be used once per run. There is some special behavior with environment variables: diff --git a/src/jinjanator/customize.py b/src/jinjanator/customize.py index 3fc6949..20a6b5d 100644 --- a/src/jinjanator/customize.py +++ b/src/jinjanator/customize.py @@ -117,11 +117,6 @@ def apply( def add_args(parser: ArgumentParser) -> ArgumentParser: """Add args to the parser""" - vargs_warning = ( - "NOTE: Due to the way python implements options, if this is" - " the last option before the template filename, you need" - " to add a -- separator before the template filename" - ) parser.add_argument( "--customize", @@ -133,20 +128,23 @@ def add_args(parser: ArgumentParser) -> ArgumentParser: parser.add_argument( "--filters", - nargs="+", + action="append", default=[], - metavar="python-file", + metavar="filters-file.py", dest="filters", - help="Load custom Jinja2 filters from a Python file(s). " - f"All top-level functions are imported as filters. {vargs_warning}", + help="Load custom Jinja2 filters from a Python file. " + "All top-level functions in provided file are imported as j2 filters. " + "(Can be used multiple times)", ) parser.add_argument( "--tests", - nargs="+", + action="append", default=[], - metavar="python-file", + metavar="tests-file.py", dest="tests", - help=f"Load custom Jinja2 tests from a Python file(s). {vargs_warning}", + help="Load custom Jinja2 tests from a Python file. " + "All top-level functions in provided file are imported as j2 tests. " + "(Can be used multiple times)", ) return parser diff --git a/tests/test_customize.py b/tests/test_customize.py index 449d4d6..d938ac3 100644 --- a/tests/test_customize.py +++ b/tests/test_customize.py @@ -46,27 +46,27 @@ def test_two_custom_filters(dir_maker: DirMakerTool) -> None: template=FileContent( "template.j2", """ - {{- "Hello, World!" | with_parens | reverse -}} + {{- key | with_parens | my_reverse -}} """, ), - data=FileContent("data.env", "key=value"), + data=FileContent("data.env", "key=Hello, World!"), filter=FileContent( "filter.py", """ - def with_parens(message): - return f"({message})" + def with_parens(message): + return f"({message})" - """, + """, ), filter2=FileContent( "filter2.py", """ - def reverse(message): - return message[::=1] + def my_reverse(message): + return message[::-1] - """, + """, ), ) @@ -74,7 +74,7 @@ def reverse(message): Path.cwd(), {}, None, - ["", "--filters", files.filter, "--", files.template, files.data], + ["", "--filters", files.filter, "--filters", files.filter2, files.template, files.data], ) @@ -108,7 +108,7 @@ def with_parens(message): Path.cwd(), {}, None, - ["", "--filter", files.filter, "--", files.template, files.data], + ["", "--filter", files.filter, files.template, files.data], ) @@ -152,7 +152,7 @@ def in_parens(message): Path.cwd(), {}, None, - ["", "--tests", files.test, "--", files.template, files.data], + ["", "--tests", files.test, files.template, files.data], ) @@ -194,7 +194,7 @@ def in_parens(message): Path.cwd(), {}, None, - ["", "--filters", files.filter, "--tests", files.test, "--", files.template, files.data], + ["", "--filters", files.filter, "--tests", files.test, files.template, files.data], ) From c3cb621051e3282623c345149704315194de8083 Mon Sep 17 00:00:00 2001 From: "Kevin P. Fleming" Date: Sun, 9 Feb 2025 07:27:59 -0500 Subject: [PATCH 7/8] Improve documentation for new features, and add Changelog entry. --- README.md | 100 ++++++++++++++++++++++++------------ changelog.d/46.adding.md | 2 + src/jinjanator/customize.py | 12 ++--- tests/test_argparse.py | 6 +-- 4 files changed, 75 insertions(+), 45 deletions(-) create mode 100644 changelog.d/46.adding.md diff --git a/README.md b/README.md index 5ded87c..afe2ad4 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,12 @@ Password: {{ "APP_PASSWORD" | env }} * `data`: (optional) path to the data used for rendering. The default is `-`: use stdin. -Options: +There is some special behavior with environment variables: + +* When `data` is not provided (data is `-`), `--format` defaults to + `env` and thus reads environment variables. + +### Options: * `--format FMT, -f FMT`: format for the data file. The default is `?`: guess from file extension. Supported formats are YAML (.yaml or @@ -155,22 +160,33 @@ Options: error will be raised). * `--version`: prints the version of the tool and the Jinja2 package installed. -Customization Options: +### Customization Options: -These options were ported from the j2cli tool for backwards compatibility (See customization section below) +For details on the behavior of these options, see the +[Customization](#customization) section. -* `--filters PYTHON_FILE` - specify a python file containing additional j2 filters as simple functions. You can use this option more than once to include multiple files. - * NOTE: while this option's behavior matches j2cli documentation, but does not match j2cli implementation. If you are migrating from j2cli and use more than one file, you will need to adjust your cli args from `... --filters file1.py file2.py ...` to `... --filters file1.py --filters file2.py ...``. +* `--filters PYTHON_FILE` - specify a file of Python source code, + containing additional Jinja2 filters as simple functions. You can + use this option more than once to include multiple files. -* `--tests PYTHON_FILE [FILE2] ...` - specify a python file containing additional j2 tests as simple functions. You can use this option more than once to include multiple files. - * NOTE: while this option's behavior matches j2cli documentation, but does not match j2cli implementation. If you are migrating from j2cli and use more than one file, you will need to adjust your cli args from `... --tests file1.py file2.py ...` to `... --tests file1.py --tests file2.py ...``. + * NOTE: While this option's behavior matches the `j2cli` + documentation, it does not match the `j2cli` implementation. If + you are migrating from `j2cli` and use more than one filters file, + you will need to specify this option once for each file. -* `--customize PYTHON_FILE` - specify a customization python file. This file can modify context, add filters/tests or change J2 configuration. Unlike `--filters` or `--tests` - this option can only be used once per run. +* `--tests PYTHON_FILE` - specify a file of Python source code, + containing additional Jinja2 tests as simple functions. You can use + this option more than once to include multiple files. -There is some special behavior with environment variables: + * NOTE: While this option's behavior matches the `j2cli` + documentation, it does not match the `j2cli` implementation. If + you are migrating from `j2cli` and use more than one tests file, + you will need to specify this option once for each file. -* When `data` is not provided (data is `-`), `--format` defaults to - `env` and thus reads environment variables. +* `--customize PYTHON_FILE` - specify a file of Python source code + containing customization functions. This file can modify the Jinja2 + context, add filters/tests, or change Jinja2's configuration. Unlike + `--filters` and `--tests`, this option can only be specified once. ## Usage Examples @@ -362,17 +378,21 @@ Pass: {{ env("USER_PASSWORD") }} Notice that there must be quotes around the environment variable name when it is a literal string. - ## Customization -(this functionality was ported from j2cli) +Jinjanator supports customizing Jinja2 template processing using two +methods - via simple files containing custom filters or tests, or via +a more advanced "customizations" file that allows you to do all of the +above as well as modify core configuration of the Jinja2 engine. -Jinjanator now supports customizing your Jinja2 template processing via two methods - via simple files containing custom filters or tests, or via a more advanced "customize" file that allows you to do all of the above as well as modify core configuration of the Jinja2 engine +### Using filters and tests files -### Via filters/tests files +The simplest way to add additional filters or tests is via "filters" +and "tests" files. These files contain Python source code consisting +of simple functions. Each function becomes a filter or test. -The simplest way to add additional filters or tests is via a "filters" or "tests" files. These files are simple python files with function. Each function becomes a filter or test. Examples: +Examples: `filters.py` @@ -409,7 +429,7 @@ And a template that uses them: The output is: ``` -$ jinjanate --filter ./filters.py --test ./tests.py -- simple.j2 +$ jinjanate --filter ./filters.py --test ./tests.py simple.j2 0 is: (even) 1 is: (odd) @@ -418,44 +438,56 @@ $ jinjanate --filter ./filters.py --test ./tests.py -- simple.j2 ``` -You can include multiple functions in each file and/or use multiple files as needed. +You can include multiple functions in each file and/or use multiple +files as needed. -### Using via a Customize File +### Using a customizations file -A more advanced way to customize your template processing is via a "customize" file. +A more advanced way to customize your template processing is by using +a "customizations" file. -Customize file allows you to: +Customizations files allow you to: -* Pass additional keywords to Jinja2 environment -* Modify the context before it's used for rendering +* Pass additional keywords to the Jinja2 environment +* Modify the context before it is used for rendering * Register custom filters and tests -This is done through *hooks* that you implement in a customization file in Python language. Each hook is a plain functions at the module level with the exact name as shown below. +This is done through *hooks* that you implement in a customization +file in Python code. Each hook is a plain function at the module +level with the exact name as shown below. The following hooks are available: * `j2_environment_params() -> dict`: returns a `dict` of additional parameters for - [Jinja2 Environment](http://jinja.pocoo.org/docs/2.10/api/#jinja2.Environment). -* `j2_environment(env: Environment) -> Environment`: lets you customize the `Environment` object. -* `alter_context(context: dict) -> dict`: lets you modify the context variables that are going to be - used for template rendering. You can do all sorts of pre-processing here. -* `extra_filters() -> dict`: returns a `dict` with extra filters for Jinja2 -* `extra_tests() -> dict`: returns a `dict` with extra tests for Jinja2 + [Jinja2 Environment](https://jinja.pocoo.org/docs/2.10/api/#jinja2.Environment). + +* `j2_environment(env: Environment) -> Environment`: lets you + customize the `Environment` object. + +* `alter_context(context: dict) -> dict`: lets you modify the context + variables that are going to be used for template rendering. You can + do all sorts of pre-processing here. + +* `extra_filters() -> dict`: returns a `dict` with extra filters for + Jinja2 + +* `extra_tests() -> dict`: returns a `dict` with extra tests for + Jinja2 All of them are optional. -The example customization.py file for your reference: +The example `customization.py file` for your reference: ```python # # Example customization.py file for jinjanator -# Contains hooks that modify the way jinjanator is initialized and used +# Contains hooks that modify the way Jinja2 is initialized and used def j2_environment_params(): """ Extra parameters for the Jinja2 Environment """ # Jinja2 Environment configuration - # http://jinja.pocoo.org/docs/2.10/api/#jinja2.Environment + # https://jinja.pocoo.org/docs/2.10/api/#jinja2.Environment return dict( # Just some examples @@ -522,8 +554,8 @@ def extra_tests(): # ``` -You can only have one customize file per run. + ## Chat If you'd like to chat with the jinjanator community, join us on diff --git a/changelog.d/46.adding.md b/changelog.d/46.adding.md new file mode 100644 index 0000000..f09e56f --- /dev/null +++ b/changelog.d/46.adding.md @@ -0,0 +1,2 @@ +Added support for '--filters', '--tests' and '--customize' from j2cli +(contributed by @mlasevich). diff --git a/src/jinjanator/customize.py b/src/jinjanator/customize.py index 20a6b5d..29e2f7c 100644 --- a/src/jinjanator/customize.py +++ b/src/jinjanator/customize.py @@ -123,7 +123,7 @@ def add_args(parser: ArgumentParser) -> ArgumentParser: default=None, metavar="python-file.py", dest="customize", - help="A Python file that implements hooks to fine-tune the jinjanator behavior", + help="A file of Python source code that implements hooks to fine-tune Jinja2 behavior", ) parser.add_argument( @@ -132,9 +132,8 @@ def add_args(parser: ArgumentParser) -> ArgumentParser: default=[], metavar="filters-file.py", dest="filters", - help="Load custom Jinja2 filters from a Python file. " - "All top-level functions in provided file are imported as j2 filters. " - "(Can be used multiple times)", + help="Load custom Jinja2 filters from a file of Python source code." + " All top-level functions in the file are imported as Jinja2 filters.", ) parser.add_argument( @@ -143,8 +142,7 @@ def add_args(parser: ArgumentParser) -> ArgumentParser: default=[], metavar="tests-file.py", dest="tests", - help="Load custom Jinja2 tests from a Python file. " - "All top-level functions in provided file are imported as j2 tests. " - "(Can be used multiple times)", + help="Load custom Jinja2 tests from file of Python source code." + " All top-level functions in the file are imported as Jinja2 tests.", ) return parser diff --git a/tests/test_argparse.py b/tests/test_argparse.py index ca03751..582abbc 100644 --- a/tests/test_argparse.py +++ b/tests/test_argparse.py @@ -42,10 +42,8 @@ def test_invalid_arg() -> None: ["-f", "env"], ["-o", "output"], ["--customize", "customize.py"], - ["--filters", "filename.py", "--"], - ["--filters", "filename.py", "filename2.py", "--"], - ["--tests", "filename.py", "--"], - ["--tests", "filename.py", "filename2.py", "--"], + ["--filters", "filename.py"], + ["--tests", "filename.py"], ], ) def test_args(args: list[str]) -> None: From 1e9d64ea1f3181e4045634b538ed91f52a1ea67f Mon Sep 17 00:00:00 2001 From: "Kevin P. Fleming" Date: Sun, 9 Feb 2025 07:32:15 -0500 Subject: [PATCH 8/8] Remove unnecessary whitespace. --- README.md | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/README.md b/README.md index afe2ad4..9ce8821 100644 --- a/README.md +++ b/README.md @@ -402,7 +402,6 @@ Examples: def parentheses(message): """ Put message in parenthesis """ return f"({message})" - ``` `tests.py` @@ -416,6 +415,7 @@ def an_odd_number(number): ``` And a template that uses them: + ``` {% for x in range(4) %} {{x}} is: {% if x is an_odd_number %} @@ -435,7 +435,6 @@ $ jinjanate --filter ./filters.py --test ./tests.py simple.j2 1 is: (odd) 2 is: (even) 3 is: (odd) - ``` You can include multiple functions in each file and/or use multiple @@ -483,7 +482,6 @@ The example `customization.py file` for your reference: # Example customization.py file for jinjanator # Contains hooks that modify the way Jinja2 is initialized and used - def j2_environment_params(): """ Extra parameters for the Jinja2 Environment """ # Jinja2 Environment configuration @@ -510,7 +508,6 @@ def j2_environment_params(): extensions=('jinja2.ext.i18n',), ) - def j2_environment(env): """ Modify Jinja2 environment @@ -522,14 +519,12 @@ def j2_environment(env): ) return env - def alter_context(context): """ Modify the context and return it """ # An extra variable context['ADD'] = '127' return context - def extra_filters(): """ Declare some custom filters. @@ -540,7 +535,6 @@ def extra_filters(): parentheses=lambda t: '(' + t + ')', ) - def extra_tests(): """ Declare some custom tests @@ -550,9 +544,6 @@ def extra_tests(): # Example: {% if a|int is custom_odd %}odd{% endif %} custom_odd=lambda n: True if (n % 2) else False ) - -# - ```