Yamlet 0.0.3: Custom tags, more built-ins, better error reporting.

This particular commit improves error reporting from deferred expression evaluation, and also adds set literals because they were a simple missing feature.

The main mission of this commit was to add custom constructor tags so that user-defined types can be built from Yamlet expressions, including string format operations. I've been mulling over what this API should look like and I'm finally pretty happy with the result.

Also fixes a couple test names... I was dropping a test here or there due to a duplicate name. Would be cool if we could somehow have warnings for that.

README.md

@@ -80,6 +80,7 @@ Here’s a summary of Yamlet’s features:
 - [Comprehensions](#comprehensions)
 - [Lambda expressions](#lambda-expressions)
 - [Custom functions](#custom-functions) (defined in Python)
+- [Custom tags](#custom-tags) for building user-defined types.
 - [GCL Special Values](#gcl-special-values) `null` and `external`.
 - Explicit [value referencing](#scoping-quirks) in composited tuples using
   `up`/`super`
@@ -401,6 +402,47 @@ With this approach, you can define custom functions for use in Yamlet
 expressions, which can lead to even more expressive configuration files.
 
 
+### Custom Tags
+
+You can add custom tag constructors to Yamlet the same way you would in Ruamel:
+
+```py
+    loader.add_constructor('!custom', CustomType)
+```
+
+In this example, `CustomType` will be instantiated with a Ruamel Loader and
+Node, which you can handle as you would with vanilla Ruamel.
+
+On top of this, however, offers an additional "style" attribute for your custom
+types:
+
+```py
+loader.add_constructor('!custom', CustomType,
+                       style=yamlet.ConstructStyle.SCALAR)
+```
+
+With this setup, `CustomType` will be instantiated using the final scalar value
+obtained from Ruamel. Additionally, Yamlet provides the following composite tags
+by default (unless `tag_compositing=False` is specified):
+
+* `!custom:fmt`: Performs a string formatting operation
+      (as usual for the Yamlet `!fmt` tag) and constructs your `CustomType`
+      with the string result.
+* `!custom:expr`: Evaluates the input as a Yamlet expression
+      (as usual for the Yamlet `!expr` tag) and constructs your `CustomType`
+      with the resulting value, whatevr type it may have.
+* `!custom:raw`: Requests the scalar value from Ruamel and constructs your
+      `CustomType` directly with that value. This is the default behavior
+      for `ConstructStyle.SCALAR`, so in this case, `!custom` and `!custom:raw`
+      behave identically.
+
+You can also specify `ConstructStyle.FMT` or `ConstructStyle.EXPR` when
+registering the constructor to set the default behavior of the base tag
+(`!custom`) to formatting or expression evaluation. All three composite tags
+(`:fmt`, `:expr`, and `:raw`) will still be available by default unless you
+set `tag_compositing=False`.
+
+
 ### GCL Special Values
 
 In addition to `up` and `super`, GCL defines the special values `null` and

tests.py

@@ -428,6 +428,38 @@ def test_array_comprehension_square(self):
         '3 green fish', '3 blue fish', '3 yellow fish',
         '4 red fish', '4 green fish', '4 yellow fish'])
 
+  def test_dict_literal(self):
+    YAMLET = '''# Yamlet
+    four: 4
+    mydict: !expr |
+      {1: 2, three: four}
+    '''
+    loader = yamlet.Loader(self.Opts())
+    t = loader.load(YAMLET)
+    self.assertEqual(set(t['mydict'].keys()), {1, 'three'})
+    self.assertEqual(t['mydict'][1], 2)
+    self.assertEqual(t['mydict']['three'], 4)
+
+  def test_set_literal(self):
+    YAMLET = '''# Yamlet
+    four: 4
+    myset: !expr |
+      {1, 2, 'three', four}
+    '''
+    loader = yamlet.Loader(self.Opts())
+    t = loader.load(YAMLET)
+    self.assertEqual(t['myset'], {1, 2, 'three', 4})
+
+  def test_pytuple_literal(self):
+    YAMLET = '''# Yamlet
+    four: 4
+    my_python_tuple: !expr |
+      (1, 2, 'three', four)
+    '''
+    loader = yamlet.Loader(self.Opts())
+    t = loader.load(YAMLET)
+    self.assertEqual(t['my_python_tuple'], (1, 2, 'three', 4))
+
 
 @ParameterizedOnOpts
 class TestFunctions(unittest.TestCase):
@@ -468,7 +500,7 @@ def test_cond_routine(self):
     self.assertEqual(y['t2']['color'], 'red')
     self.assertEqual(y['t3']['color'], 'green')
 
-  def test_cond_routine(self):
+  def test_cond_routine_2(self):
     YAMLET = '''# Yamlet
     t1:
       conditionals: !expr |
@@ -535,7 +567,7 @@ def test_if_statement_templating(self):
     self.assertEqual(set(y['t3'].keys()), {'animal', 'environment'})
     self.assertEqual(set(y['t4'].keys()), {'animal', 'recommendation'})
 
-  def test_if_statement_templating(self):
+  def test_if_statement_templating_2(self):
     YAMLET = '''# Yamlet
     t0:
       !if animal == 'fish':
@@ -929,7 +961,7 @@ def test_specializing_conditions(self):
     self.assertEqual(y['flashy']['variable'], 'specialized value')
     self.assertEqual(y['boring']['variable'], 'defaulted value')
 
-  def test_specializing_conditions(self):
+  def test_specializing_conditions_2(self):
     YAMLET = '''# Yamlet
     tp:
       variable: !rel defaulted value
@@ -1315,5 +1347,126 @@ def test_one_fish_two_fish_from_readme(self):
     self.assertEqual(t['fishes'], '1 fish, 2 fish, red fish, blue fish')
 
 
+@ParameterizedOnOpts
+class TestCustomConstructors(unittest.TestCase):
+  '''
+  XXX: Ruamel's constructor object is static and inherited between all YAML()
+  instances, so this class tries to use a unique name for each test's tags
+  to avoid cross-contamination.
+  '''
+  class CustomType_LN:
+    def __init__(self, loader, node): self.value = loader.construct_scalar(node)
+    def __str__(self): return self.value
+    def __repr__(self): return f'CustomType_LN({self.value})'
+    def __eq__(self, other):
+      return (isinstance(other, TestCustomConstructors.CustomType_LN)
+              and other.value == self.value)
+
+  class CustomType_V:
+    def __init__(self, value): self.value = value
+    def __str__(self): return self.value
+    def __repr__(self): return f'CustomType_V({self.value})'
+    def __eq__(self, other):
+      return (isinstance(other, TestCustomConstructors.CustomType_V)
+              and other.value == self.value)
+
+  def test_composited_tags(self):
+    YAMLET = '''# Yamlet
+    one: 1
+    two: 2
+    case1: !custom1 one + two
+    case2: !custom1:fmt '{one} + {two}'
+    case3: !custom1:expr one + two
+    '''
+    loader = yamlet.Loader(self.Opts())
+    loader.add_constructor('!custom1', self.CustomType_V,
+                           style=yamlet.ConstructStyle.SCALAR)
+    t = loader.load(YAMLET)
+    self.assertIsInstance(t['case1'], self.CustomType_V)
+    self.assertIsInstance(t['case2'], self.CustomType_V)
+    self.assertIsInstance(t['case3'], self.CustomType_V)
+    self.assertEqual(t['case1'].value, 'one + two')
+    self.assertEqual(t['case2'].value, '1 + 2')
+    self.assertEqual(t['case3'].value, 3)
+
+  def test_raw_tag(self):
+    YAMLET = '''# Yamlet
+    one: 1
+    two: 2
+    case1: !custom2 one + two
+    '''
+    loader = yamlet.Loader(self.Opts())
+    loader.add_constructor('!custom2', self.CustomType_LN)
+    t = loader.load(YAMLET)
+    self.assertIsInstance(t['case1'], self.CustomType_LN)
+    self.assertEqual(t['case1'].value, 'one + two')
+
+  def test_expr_style_tag(self):
+    YAMLET = '''# Yamlet
+    one: 1
+    two: 2
+    case1: !custom3 one + two
+    case2: !custom3:raw one + two
+    case3: !custom3:fmt '{one} + {two}'
+    case4: !custom3:expr one + two
+    '''
+    loader = yamlet.Loader(self.Opts())
+    loader.add_constructor('!custom3', self.CustomType_V,
+                           style=yamlet.ConstructStyle.EXPR)
+    t = loader.load(YAMLET)
+    self.assertIsInstance(t['case1'], self.CustomType_V)
+    self.assertIsInstance(t['case2'], self.CustomType_V)
+    self.assertIsInstance(t['case3'], self.CustomType_V)
+    self.assertIsInstance(t['case4'], self.CustomType_V)
+    self.assertEqual(t['case1'].value, 3)
+    self.assertEqual(t['case2'].value, 'one + two')
+    self.assertEqual(t['case3'].value, '1 + 2')
+    self.assertEqual(t['case4'].value, 3)
+
+  def test_fmt_style_tag(self):
+    YAMLET = '''# Yamlet
+    one: 1
+    two: 2
+    case1: !custom4 '{one} + {two}'
+    case2: !custom4:raw one + two
+    case3: !custom4:fmt '{one} + {two}'
+    case4: !custom4:expr one + two
+    '''
+    loader = yamlet.Loader(self.Opts())
+    loader.add_constructor('!custom4', self.CustomType_V,
+                           style=yamlet.ConstructStyle.FMT)
+    t = loader.load(YAMLET)
+    self.assertIsInstance(t['case1'], self.CustomType_V)
+    self.assertIsInstance(t['case2'], self.CustomType_V)
+    self.assertIsInstance(t['case3'], self.CustomType_V)
+    self.assertIsInstance(t['case4'], self.CustomType_V)
+    self.assertEqual(t['case1'].value, '1 + 2')
+    self.assertEqual(t['case2'].value, 'one + two')
+    self.assertEqual(t['case3'].value, '1 + 2')
+    self.assertEqual(t['case4'].value, 3)
+
+  def test_fancy_ctor_in_opts(self):
+    YAMLET = '''# Yamlet
+    one: 1
+    two: 2
+    case1: !custom5 '{one} + {two}'
+    case2: !custom5:raw one + two
+    case3: !custom5:fmt '{one} + {two}'
+    case4: !custom5:expr one + two
+    '''
+    loader = yamlet.Loader(self.Opts(constructors={
+        '!custom5': {'ctor': self.CustomType_V,
+                     'style': yamlet.ConstructStyle.FMT}}))
+    t = loader.load(YAMLET)
+    self.assertIsInstance(t['case1'], self.CustomType_V)
+    self.assertIsInstance(t['case2'], self.CustomType_V)
+    self.assertIsInstance(t['case3'], self.CustomType_V)
+    self.assertIsInstance(t['case4'], self.CustomType_V)
+    self.assertEqual(t['case1'].value, '1 + 2')
+    self.assertEqual(t['case2'].value, 'one + two')
+    self.assertEqual(t['case3'].value, '1 + 2')
+    self.assertEqual(t['case4'].value, 3)
+
+
 if __name__ == '__main__':
   unittest.main()

yamlet.py

@@ -54,6 +54,21 @@ def __init__(self, import_resolver=None, missing_name_value=Error,
     self.debugging = _yamlet_debug_opts or _DebugOpts()
 
 
+class ConstructStyle:
+  # Your type will be constructed with the loader and node.
+  # This is identical to how Ruamel/PyYAML constructors work.
+  RAW = 'RAW'
+  # Your type will be constructed with the processed YAML value.
+  # This may be a string, list, GclDict, etc.
+  SCALAR = 'SCALAR'
+  # The value will be treated as a string format operation (`!fmt`),
+  # and your type will be constructed from its result.
+  FMT = 'FMT'
+  # The value will be treated as a Yamlet expression (`!expr`),
+  # and your type will be constructed from the result.
+  EXPR = 'EXPR'
+
+
 class Loader(ruamel.yaml.YAML):
   def __init__(self, opts=None):
     super().__init__()
@@ -102,17 +117,48 @@ def Constructor(loader, node):
     yc.add_constructor("!null",      ConstructConstant('null',     null))
     yc.add_constructor("!external",  ConstructConstant('external', external))
     for tag, ctor in self.yamlet_options.constructors.items():
-      self.add_constructor(tag, ctor)
+      if callable(ctor): self.add_constructor(tag, ctor)
+      else:
+        assert isinstance(ctor, dict), ('Yamlet constructors should be callable'
+            ' or give arguments to `add_constructor`; '
+            f'got `{type(ctor).__name__}` for `{tag}`')
+        self.add_constructor(tag, **ctor)
 
-  def add_constructor(self, tag, ctor):
-    def UserConstructor(loader, node):
-      try: return ctor(loader, node)
-      except Exception as e:
-        raise ConstructorError(None, None,
-            f'Yamlet user constructor `!{tag}` encountered an error; '
-            'is your type constructable from `(ruamel.Loader, ruamel.Node)`?',
-            node.start_mark) from e
-    self.constructor.add_constructor(tag, UserConstructor)
+  def add_constructor(self, tag, ctor,
+                      style=ConstructStyle.RAW, tag_compositing=True):
+    yc = self.constructor
+    if style == ConstructStyle.RAW:
+      def RawUserConstructor(loader, node):
+        try: return ctor(loader, node)
+        except Exception as e:
+          raise ConstructorError(None, None,
+              f'Yamlet user constructor `!{tag}` encountered an error; '
+              'is your type constructable from `(ruamel.Loader, ruamel.Node)`?',
+              node.start_mark) from e
+      yc.add_constructor(tag, RawUserConstructor)
+      return self
+    def ConstructDefer(deferred, tp):
+      def Constructor(loader, node):
+        return deferred(tp, loader.construct_scalar(node),
+                        YamlPoint(node.start_mark, node.end_mark))
+      return Constructor
+    def ConstructScalar(tp):
+      def Constructor(loader, node): return tp(loader.construct_scalar(node))
+      return Constructor
+    match style:
+      case ConstructStyle.SCALAR:
+        yc.add_constructor(tag, ConstructScalar(ctor))
+      case ConstructStyle.FMT:
+        yc.add_constructor(tag, ConstructDefer(StringToSubAndWrap, ctor))
+      case ConstructStyle.EXPR:
+        yc.add_constructor(tag, ConstructDefer(ExprToEvalAndWrap, ctor))
+      case _:
+        raise ValueError(f'Unknown construction style `{style}`')
+    if tag_compositing:
+      yc.add_constructor(tag + ':raw', ConstructScalar(ctor))
+      yc.add_constructor(tag + ':fmt', ConstructDefer(StringToSubAndWrap, ctor))
+      yc.add_constructor(tag + ':expr', ConstructDefer(ExprToEvalAndWrap, ctor))
+    return self
 
   def LoadCachedFile(self, fn):
     fn = fn.resolve()
@@ -418,7 +464,7 @@ def __str__(self): return self.traced_message
   return YamletException
 
 
-class ExceptionWithYamletTrace(Exception):
+class ExceptionWithYamletTrace(YamletBaseException):
   def __init__(self, ex_class, message):
     super().__init__(message)
     self.ex_class = ex_class
@@ -624,20 +670,33 @@ def _gcl_evaluate_(self, value, ectx):
     return _ResolveStringValue(value, ectx)
 
 
-class TupleListToComposite(DeferredValue):
+class ExpressionToEvaluate(DeferredValue):
   def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs)
   def _gcl_explanation_(self):
-    return f'Compositing tuple list `{self._gcl_construct_}`'
+    return f'Evaluating expression `{self._gcl_construct_.strip()}`'
   def _gcl_evaluate_(self, value, ectx):
-      return _CompositeYamlTupleList(value, ectx)
+    try: return _GclExprEval(value, ectx)
+    except Exception as e:
+      if isinstance(e, YamletBaseException): raise
+      ectx.Raise(type(e), f'Error in Yamlet expression: {e}.\n', e)
 
 
-class ExpressionToEvaluate(DeferredValue):
+class DeferredValueWrapper(DeferredValue):
+  def __init__(self, klass, *args, **kwargs):
+    super().__init__(*args, **kwargs)
+    self.klass = klass
+  def _gcl_evaluate_(self, value, ectx):
+    return self.klass(super()._gcl_evaluate_(value, ectx))
+class StringToSubAndWrap(DeferredValueWrapper, StringToSubstitute): pass
+class ExprToEvalAndWrap(DeferredValueWrapper, ExpressionToEvaluate): pass
+
+
+class TupleListToComposite(DeferredValue):
   def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs)
   def _gcl_explanation_(self):
-    return f'Evaluating expression `{self._gcl_construct_.strip()}`'
+    return f'Compositing tuple list `{self._gcl_construct_}`'
   def _gcl_evaluate_(self, value, ectx):
-    return _GclExprEval(value, ectx)
+      return _CompositeYamlTupleList(value, ectx)
 
 
 class IfLadderTableIndex(DeferredValue):
@@ -1307,14 +1366,18 @@ def EvalGclAst(et, ectx):
     case ast.Tuple:
       return tuple(ev(x) for x in et.elts)
 
+    case ast.Set:
+      return set(ev(elt) for elt in et.elts)
+
     case ast.Dict:
       def EvalKey(k):
         if isinstance(k, ast.Name): return k.id
         if isinstance(k, ast.Constant):
           if isinstance(k.value, str):
             return _ResolveStringValue(k.value, ectx)
+          return k.value
         ectx.Raise(KeyError, 'Yamlet keys should be names or strings. '
-                             f'Got {type(et).__name__}')
+                             f'Got `{type(k).__name__}`:\n{k}')
       children = []
       def DeferAst(v):
         if isinstance(v, ast.Dict):