Add comprehensions, tuples, more builtins.

Adds a lot more built-in types; pretty much every safe-looking Python builtin is available now. No support for `pow`/`**` as these allow filling memory too quickly and character-efficiently.

Comprehensions can be used in generator expressions and for constructing dicts, lists, and sets. May need to change dict generator output type to GclTuple, but for now, those and `dict()` return normal Python dicts.

README.md

@@ -77,6 +77,7 @@ Here’s a summary of Yamlet’s features:
 - [Conditionals, as seen in procedural languages](#conditionals)
 - [File Imports](#file-imports)
   (to allow splitting up configs or splitting out templates)
+- [Comprehensions](#comprehensions)
 - [Lambda expressions](#lambda-expressions)
 - [Custom functions](#custom-functions) (defined in Python)
 - [GCL Special Values](#gcl-special-values) `null` and `external`.
@@ -338,6 +339,25 @@ may import as many files as you like, import files that don't exist, import
 yourself, or import files cyclically—errors will only occur if you try to
 access undefined or cyclic values within those files.
 
+### Comprehensions
+
+Yamlet expressions inherit list comprehension syntax from Python.
+
+```yaml
+my_array: [1, 2, 'red', 'blue']
+fishes: !expr r', '.join('{x} fish' for x in my_array)
+```
+
+In this example, the `fishes` array evaluates to
+`1 fish, 2 fish, red fish, blue fish`.
+
+A couple notes:
+1. A raw string is used for the comma character to stop YAML
+   from interpreting just the first literal as the scalar value.
+2. Though an f-string could be used in the generator expression,
+   it is not required as all Yamlet literals use {} for formatting.
+   Using an f-string would allow Python's formatting options in the {}.
+
 ### Lambda Expressions
 
 Lambda expressions in Yamlet are read in from YAML as normal strings, then

tests.py

@@ -376,7 +376,7 @@ def test_invalid_up_super_usage(self):
 
 
 @ParameterizedOnOpts
-class TestStringMechanics(unittest.TestCase):
+class TestValueMechanics(unittest.TestCase):
   def test_escaped_braces(self):
     YAMLET = '''# Yamlet
     v: Hello
@@ -387,6 +387,47 @@ def test_escaped_braces(self):
     y = loader.load(YAMLET)
     self.assertEqual(y['v3'], '{Hello}, {{world}}{s}!')
 
+  def test_array_comprehension(self):
+    YAMLET = '''# Yamlet
+    my_array: [1, 2, 'red', 'blue']
+    fishes: !expr "['{x} fish' for x in my_array]"
+    '''
+    loader = yamlet.Loader(self.Opts())
+    t = loader.load(YAMLET)
+    self.assertEqual(t['fishes'], ['1 fish', '2 fish', 'red fish', 'blue fish'])
+
+  def test_dict_comprehension(self):
+    YAMLET = '''# Yamlet
+    my_array: [1, 2, 'red', 'blue']
+    fishes: !expr "{x: 'fish' for x in my_array}"
+    '''
+    loader = yamlet.Loader(self.Opts())
+    t = loader.load(YAMLET)
+    self.assertEqual(t['fishes'],
+                     {1: 'fish', 2: 'fish', 'red': 'fish', 'blue': 'fish'})
+
+  def test_array_comprehension_square(self):
+    YAMLET = '''# Yamlet
+    array1: [1, 2, 3, 4]
+    array2: ['red', 'green', 'blue', 'yellow']
+    fishes: !expr "['{x} {y} fish' for x in array1 for y in array2]"
+    filtered: !expr |
+        ['{x} {y} fish' for x in array1 for y in array2 if x != len(y)]
+    '''
+    loader = yamlet.Loader(self.Opts())
+    t = loader.load(YAMLET)
+    fishes = t['fishes']
+    self.assertEqual(len(fishes), 16)
+    self.assertEqual(fishes[0], '1 red fish')
+    self.assertEqual(fishes[15], '4 yellow fish')
+    filtered = t['filtered']
+    self.assertEqual(len(filtered), 14)
+    self.assertEqual(filtered, [
+        '1 red fish', '1 green fish', '1 blue fish', '1 yellow fish',
+        '2 red fish', '2 green fish', '2 blue fish', '2 yellow fish',
+        '3 green fish', '3 blue fish', '3 yellow fish',
+        '4 red fish', '4 green fish', '4 yellow fish'])
+
 
 @ParameterizedOnOpts
 class TestFunctions(unittest.TestCase):
@@ -1264,6 +1305,15 @@ def test_easy_load_file(self):
     self.assertEqual(t['childtuple2']['coolbeans'],
                      'Hello, world! I say awesome sauce!')
 
+  def test_one_fish_two_fish_from_readme(self):
+    YAMLET = '''# Yamlet
+    my_array: [1, 2, 'red', 'blue']
+    fishes: !expr r', '.join('{x} fish' for x in my_array)
+    '''
+    loader = yamlet.Loader(self.Opts())
+    t = loader.load(YAMLET)
+    self.assertEqual(t['fishes'], '1 fish, 2 fish, red fish, blue fish')
+
 
 if __name__ == '__main__':
   unittest.main()

yamlet.py

@@ -30,8 +30,9 @@
 import sys
 import token
 import tokenize
+import typing
 
-VERSION = '0.0.2'
+VERSION = '0.0.3'
 ConstructorError = ruamel.yaml.constructor.ConstructorError
 class YamletBaseException(Exception): pass
 
@@ -85,15 +86,6 @@ def Constructor(loader, node):
             node.start_mark)
         return val
       return Constructor
-    def UserConstructor(ctor):
-      def Construct(loader, node):
-        try: return ctor(loader, node)
-        except Exception as e:
-          raise ConstructorError(None, None,
-              f'Yamlet user constructor `!{tag}` encountered an error; '
-              'is your type constructable from `(ruamel.Loader, ruamel.Node)`?',
-              node.start_mark) from e
-      return Construct
 
     yc = self.constructor
     yc.add_constructor(None, UndefinedConstructor)  # Raise on undefined tags
@@ -110,7 +102,17 @@ def Construct(loader, node):
     yc.add_constructor("!null",      ConstructConstant('null',     null))
     yc.add_constructor("!external",  ConstructConstant('external', external))
     for tag, ctor in self.yamlet_options.constructors.items():
-      yc.add_constructor(tag, UserConstructor(ctor))
+      self.add_constructor(tag, ctor)
+
+  def add_constructor(self, tag, ctor):
+    def UserConstructor(loader, node):
+      try: return ctor(loader, node)
+      except Exception as e:
+        raise ConstructorError(None, None,
+            f'Yamlet user constructor `!{tag}` encountered an error; '
+            'is your type constructable from `(ruamel.Loader, ruamel.Node)`?',
+            node.start_mark) from e
+    self.constructor.add_constructor(tag, UserConstructor)
 
   def LoadCachedFile(self, fn):
     fn = fn.resolve()
@@ -127,9 +129,15 @@ def LoadCachedFile(self, fn):
     return res
 
   def ConstructGclDict(self, loader, node):
-    return ProcessYamlPairs(
-        loader.construct_pairs(node), gcl_opts=self.yamlet_options,
-        yaml_point=YamlPoint(start=node.start_mark, end=node.end_mark))
+    try:
+      return ProcessYamlPairs(
+          loader.construct_pairs(node), gcl_opts=self.yamlet_options,
+          yaml_point=YamlPoint(start=node.start_mark, end=node.end_mark))
+    except Exception as e:
+      if isinstance(e, ConstructorError): raise
+      raise ConstructorError(None, None,
+          f'Yamlet error while processing dictionary: {str(e)}',
+          node.start_mark) from e
 
   def DeferGclComposite(self, loader, node):
     marks = YamlPoint(node.start_mark, node.end_mark)
@@ -913,7 +921,10 @@ def ProcessYamlPairs(mapping_pairs, gcl_opts, yaml_point):
   filtered_pairs = {}
   preprocessors = {}
   if_directive = None
-  cErr = lambda msg, v: ConstructorError(None, None, msg, v._yaml_point_.start)
+  cErr = lambda msg, v=None: ConstructorError(
+             None, None, msg,
+             v._yaml_point_.start if hasattr(v, '_yaml_point_')
+                                  else yaml_point.start)
   notDict = lambda v: (
       not isinstance(v, GclDict) and not isinstance(v, PreprocessingTuple))
   notDictErr = lambda k, v: cErr(
@@ -949,6 +960,9 @@ def terminateIfDirective():
       raise cErr('Yamlet keys from YAML mappings must be constant', k)
     else:
       terminateIfDirective()
+      if not isinstance(k, typing.Hashable):
+        raise cErr(f'found unacceptable key (unhashable type: \''
+                   f'{type(k).__name__}\'): {k}')
       if k in filtered_pairs:
         raise cErr(f'Duplicate tuple key `{k}`: '
                    'this is defined to be an error in Yamlet 0.0')
@@ -1092,7 +1106,17 @@ def _BuiltinFuncsMapper():
   def cond(ectx, condition, if_true, if_false):
     return (EvalGclAst(if_true, ectx) if EvalGclAst(condition, ectx)
             else EvalGclAst(if_false, ectx))
-  return { 'cond': cond, 'len': len, 'int': int, 'float': float, 'str': str }
+  # XXX: I elided next() because it's ugly and throws; probably better to add
+  # an nth() method that just returns the nth item in the sequence, or maybe
+  # just pile itertools onto the stack.
+  python_builtins = [
+      abs, all, any, ascii, bin, bool, bytearray, bytes, callable, chr, complex,
+      dict, divmod, enumerate, filter, float, format, frozenset, getattr,
+      hasattr, hash, hex, id, int, isinstance, issubclass, iter, len, list, map,
+      max, min, oct, ord, range, repr, reversed, round, set, setattr, slice,
+      sorted, str, sum, tuple, type, vars, zip
+  ]
+  return { 'cond': cond } | {bi.__name__: bi for bi in python_builtins}
 _BUILTIN_FUNCS = _BuiltinFuncsMapper()
 
 
@@ -1128,6 +1152,42 @@ def _GclExprEval(expr, ectx):
   return EvalGclAst(_InsertCompositOperators(expr), ectx)
 
 
+def _EvalComprehension(elt, generators, ectx, index=0):
+  """Evaluates nested generators with optional if-clauses in a comprehension."""
+  # Base case: all generators processed, evaluate `elt`
+  if index == len(generators):
+    yield EvalGclAst(elt, ectx)
+    return
+
+  generator = generators[index]
+  iter_values = EvalGclAst(generator.iter, ectx)
+
+  ectx.Assert(isinstance(iter_values, typing.Iterable),
+              f'Expected an iterable in generator expression, got `{type(iter_values).__name__}`.', TypeError)
+  ectx.Assert(isinstance(generator.target, (ast.Name, ast.Tuple)),
+              f'Comprehension target should be Name or Tuple, '
+              f'but got `{type(generator.target).__name__}`', TypeError)
+
+  for item in iter_values:
+    # Set up target variables in a new scope
+    if isinstance(generator.target, ast.Name):
+      targets = {generator.target.id: item}
+    else:
+      ectx.Assert(all(isinstance(t, ast.Name) for t in generator.target.elts),
+                  'All tuple entries in comprehension target should be names.',
+                  TypeError)
+      ectx.Assert(len(generator.target.elts) == len(item),
+                  'Tuple unpacking length mismatch in comprehension target.',
+                  TypeError)
+      targets = {t.id: item[i] for i, t in enumerate(generator.target.elts)}
+
+    scoped_ectx = ectx.Branch('comprehension', ectx.GetPoint(),
+                              ectx.NewGclDict(targets))
+
+    if all(EvalGclAst(cond, scoped_ectx) for cond in generator.ifs):
+      yield from _EvalComprehension(elt, generators, scoped_ectx, index + 1)
+
+
 def EvalGclAst(et, ectx):
   ev = lambda x: EvalGclAst(x, ectx)
   match type(et):
@@ -1137,14 +1197,31 @@ def EvalGclAst(et, ectx):
       if isinstance(et.value, str):
         return _ResolveStringValue(et.value, ectx)
       return et.value
+
+    case ast.JoinedStr:
+      return ''.join(ev(v) for v in et.values)
+
+    case ast.FormattedValue:
+      v = ev(et.value)
+      match et.conversion:
+        case -1:  v = f'{v}'  # XXX: documentation does not say what this is
+        case 115: v = str(v)
+        case 114: v = repr(v)
+        case 97:  v = ascii(v)
+        case _: ectx.Raise(f'Unsupported Python conversion {et.conversion}')
+      if not et.format_spec: return v
+      return ev(et.format_spec).format(v)
+
     case ast.Attribute:
       val = ev(et.value)
       if et.attr in _BUILTIN_NAMES:
         with ectx.Scope(val): return _BUILTIN_NAMES[et.attr](ectx)
       if isinstance(val, GclDict):
         try: return val._gcl_traceable_get_(et.attr, ectx)
         except KeyError: ectx.Raise(KeyError, f'No {et.attr} in this scope.')
-      try: return val[et.attr]
+      try:
+        if isinstance(val, GclDict): return val[et.attr]
+        else: return getattr(val, et.attr)
       except Exception as e:
         ectx.Raise(KeyError, f'Cannot access attribute on value:\n  value'
              f'({type(val).__name__}): {val}\n  attribute: {et.attr}\n', e)
@@ -1159,6 +1236,7 @@ def EvalGclAst(et, ectx):
         case ast.Mod: return l % r
         case ast.MatMult: return _CompositeGclTuples([l, r], ectx)
       ectx.Raise(NotImplementedError, f'Unsupported binary operator `{et.op}`.')
+
     case ast.Compare:
       l = ev(et.left)
       for op, r in zip(et.ops, et.comparators):
@@ -1179,6 +1257,7 @@ def EvalGclAst(et, ectx):
         if not v: return False
         l = r
       return True
+
     case ast.BoolOp:
       v = None
       match type(et.op):
@@ -1193,8 +1272,10 @@ def EvalGclAst(et, ectx):
         case _: ectx.Raise(NotImplementedError,
                            f'Unknown boolean operator `{op}`.')
       return v
+
     case ast.IfExp:
       return ev(et.body) if ev(et.test) else ev(et.orelse)
+
     case ast.Call:
       fun, fun_name = None, None
       if isinstance(et.func, ast.Name):
@@ -1212,14 +1293,20 @@ def EvalGclAst(et, ectx):
       fun_args = [EvalGclAst(arg, ectx) for arg in fun_args]
       fun_kwargs = {k: EvalGclAst(v, ectx) for k, v in fun_kwargs.items()}
       return fun(*fun_args, **fun_kwargs)
+
     case ast.Subscript:
       v = ev(et.value)
       if isinstance(et.slice, ast.Slice):
         return v[et.slice.lower and ev(et.slice.lower)
                 :et.slice.upper and ev(et.slice.upper)]
       return v[ev(et.slice)]
+
     case ast.List:
       return [ev(x) for x in et.elts]
+
+    case ast.Tuple:
+      return tuple(ev(x) for x in et.elts)
+
     case ast.Dict:
       def EvalKey(k):
         if isinstance(k, ast.Name): return k.id
@@ -1239,6 +1326,17 @@ def DeferAst(v):
                              for k,v in zip(et.keys, et.values)})
       for c in children: c._gcl_parent_ = res
       return res
+
+    case ast.GeneratorExp:
+      return _EvalComprehension(et.elt, et.generators, ectx)
+    case ast.ListComp:
+      return list(_EvalComprehension(et.elt, et.generators, ectx))
+    case ast.SetComp:
+      return set(_EvalComprehension(et.elt, et.generators, ectx))
+    case ast.DictComp:
+      return dict(_EvalComprehension(ast.Tuple([et.key, et.value]),
+                                     et.generators, ectx))
+
   ectx.Raise(NotImplementedError,
              f'Undefined Yamlet operation `{type(et).__name__}`')