CRITICAL: Correct copy bug in YamletIfElseLadder; also check super sc…

…opes in name lookup.

Fixes an extremely weird error that I had previously been working around by duplicating the result tuple for each new element merged, during a `_CompositeGclTuples` operation. The code had somehow grown to rely on two copies being performed, which kind of screwed up the super mechanism (or at least consumed undue extra memory and left the `super` chain needlessly muddy).

The `super` chain is now comparatively neat, fit, and trim. Like me, I guess; I've been taking care of myself. Mostly. Thank you for asking.

Also adds checking for said fit and trim super scopes during name lookup, because while all variables in the destination hierarchy should take precedence over variables in the super hierarchy, those scopes should still be fallen back on before hitting module-level globals.

It may actually be preferable to hit module-level variables in the current scope before checking them in super,  but I'll need a use case first. The remedy there would be to stop treating module globals as special and just give file-scope variables a proper parent. I'll get around to it.

tests.py

@@ -50,6 +50,7 @@ def active(v):
     ParameterizedOnOpts if active(os.getenv('yamlet_stress'))
     else DefaultConfigOnly)
 
+
 @ParameterizedOnOpts
 class TestTupleCompositing(unittest.TestCase):
   def test_composited_fields(self):
@@ -481,6 +482,133 @@ def test_string_from_up_in_if(self):
     t = loader.load(YAMLET)
     self.assertEqual(t['t']['val2'], 1337)
 
+  def test_reference_other_scope(self):
+    YAMLET = '''# Yamlet
+    context:
+      not_in_evaluating_scope: Hello, world!
+      referenced: !fmt '{not_in_evaluating_scope}'
+    result: !expr context.referenced
+    '''
+    loader = yamlet.Loader(self.Opts())
+    t = loader.load(YAMLET)
+    self.assertEqual(t['result'], 'Hello, world!')
+
+  def test_reference_other_scope_2(self):
+    YAMLET = '''# Yamlet
+    context:
+      not_in_evaluating_scope: Hello, world!
+      referenced: !fmt '{not_in_evaluating_scope}'
+    context2:
+      inner_ref: !expr context
+      referenced_2: !expr inner_ref.referenced
+    result: !expr context2.referenced_2
+    '''
+    loader = yamlet.Loader(self.Opts())
+    t = loader.load(YAMLET)
+    self.assertEqual(t['result'], 'Hello, world!')
+
+  def test_reference_env(self):
+    YAMLET = '''# Yamlet
+    other_context:
+      not_inherited: Hello, world!
+      referenced: !fmt '{not_inherited}'
+    my_context:
+      my_variable: !expr other_context.referenced
+    '''
+    loader = yamlet.Loader(self.Opts())
+    t = loader.load(YAMLET)
+    self.assertEqual(t['my_context']['my_variable'], 'Hello, world!')
+
+  def test_reference_nested_env(self):
+    YAMLET = '''# Yamlet
+    other_context:
+      not_inherited: Hello, world!
+      subcontext:
+        referenced: !fmt '{not_inherited}'
+    my_context:
+      captured_subcontext: !expr other_context.subcontext
+    '''
+    loader = yamlet.Loader(self.Opts())
+    t = loader.load(YAMLET)
+    self.assertEqual(t['my_context']['captured_subcontext']['referenced'],
+                     'Hello, world!')
+
+  def test_reference_nested_env_2(self):
+    YAMLET = '''# Yamlet
+    other_context:
+      not_inherited: Hello, world!
+      subcontext:
+        referenced: !fmt '{not_inherited}'
+    my_context:
+      captured_subcontext: !composite
+        - other_context.subcontext
+        - red: herring
+          not_inherited: 'Good night, moon!'
+    '''
+    loader = yamlet.Loader(self.Opts())
+    t = loader.load(YAMLET)
+    self.assertEqual(t['my_context']['captured_subcontext']['referenced'],
+                     'Good night, moon!')
+
+  def test_reference_nested_env_3(self):
+    YAMLET = '''# Yamlet
+    other_context:
+      not_inherited: Hello, world!
+      subcontext:
+        referenced: !fmt '{not_inherited}'
+    my_context:
+      not_inherited: 'Good night, moon!'
+      captured_subcontext: !composite
+        - other_context.subcontext
+        - red: herring
+    '''
+    loader = yamlet.Loader(self.Opts())
+    t = loader.load(YAMLET)
+    self.assertEqual(t['my_context']['captured_subcontext']['referenced'],
+                     'Good night, moon!')
+
+  def test_reference_nested_env_4(self):
+    YAMLET = '''# Yamlet
+    other_context:
+      not_inherited: Hello, world!
+      subcontext:
+        referenced: !fmt '{not_inherited}'
+    my_context:
+      captured_subcontext: !composite
+        - other_context.subcontext
+        - red: herring
+    test_probe: !expr my_context.captured_subcontext.super
+    '''
+    loader = yamlet.Loader(self.Opts())
+    t = loader.load(YAMLET)
+    self.assertTrue(t['test_probe'] is t['other_context']['subcontext'])
+    self.assertEqual(t['my_context']['captured_subcontext']['referenced'],
+                     'Hello, world!')
+
+  def test_reference_nested_env_5(self):
+    YAMLET = '''# Yamlet
+    chain_1:
+      not_inherited: Hello, world!
+      subcontext:
+        referenced: !fmt '{not_inherited}'
+    chain_2:
+      captured_subcontext_1: !composite
+        - chain_1.subcontext
+        - red: herring
+    chain_3:
+      captured_subcontext_2: !composite
+        - chain_2.captured_subcontext_1
+        - hoax: value
+    chain_4:
+      captured_subcontext_3: !composite
+        - chain_3.captured_subcontext_2
+        - artifice: more junk
+    result: !fmt '{chain_4.captured_subcontext_3.referenced}'
+    '''
+    loader = yamlet.Loader(self.Opts())
+    t = loader.load(YAMLET)
+    self.assertEqual(t['result'], 'Hello, world!')
+
 
 @ParameterizedOnOpts
 class TestFunctions(unittest.TestCase):
@@ -967,6 +1095,8 @@ def __eq__(self, other):
       if isinstance(other, str): return self.val == other
       return self.val == other.val
 
+    def __repr__(self): return f'RelativeStringValue({self.val!r})'
+
   def test_specializing_conditions(self):
     YAMLET = '''# Yamlet
     tp:

yamlet.py

@@ -21,7 +21,6 @@
 # SOFTWARE.
 
 import ast
-import copy
 import io
 import keyword
 import pathlib
@@ -326,13 +325,13 @@ def yamlet_merge(self, other, ectx):
 
 class GclDict(dict, Compositable):
   def __init__(self, *args,
-               gcl_parent, gcl_super, gcl_opts, yaml_point, preprocessors,
-               **kwargs):
-    super().__init__(*args, **kwargs)
+               gcl_parent, gcl_super, gcl_opts, yaml_point, preprocessors):
+    super().__init__(*args)
     self._gcl_parent_ = gcl_parent
     self._gcl_super_ = gcl_super
     self._gcl_opts_ = gcl_opts
-    self._gcl_preprocessors_ = preprocessors or {}
+    assert isinstance(preprocessors, dict)
+    self._gcl_preprocessors_ = preprocessors
     self._gcl_provenances_ = {}
     self._yaml_point_ = yaml_point
 
@@ -390,8 +389,8 @@ def yamlet_merge(self, other, ectx):
                 'Expected dict-like type to composite.', ex_class=TypeError)
     for k, v in other._gcl_noresolve_items_():
       if isinstance(v, Compositable):
-        v1 = super().setdefault(k, v)
-        if v1 is not v:
+        v1 = super().setdefault(k, None)
+        if v1 is not None:
           if not isinstance(v1, Compositable):
             ectx.Raise(TypeError, f'Cannot composite `{type(v1)}` object `{k}` '
                                    'with dictionary value in extending tuple.')
@@ -544,7 +543,7 @@ def NewGclDict(self, *args, gcl_parent=None, gcl_super=None, **kwargs):
         gcl_parent=gcl_parent or self.scope,
         gcl_super=gcl_super,
         gcl_opts=self.opts,
-        preprocessors=None,
+        preprocessors={},
         yaml_point=self._trace_point)
 
   def Branch(self, name, yaml_point, scope):
@@ -659,7 +658,9 @@ def _gcl_resolve_(self, ectx):
       if ectx.opts.caching == YamletOptions.CACHE_DEBUG:
         if getattr(self, '_gcl_cache_debug_', _empty) is not _empty:
           assert res == self._gcl_cache_debug_, ('Internal error: Cache bug! '
-              f'Cached value `{self._gcl_cache_debug_}` is not `{res}`!')
+              f'Cached value `{self._gcl_cache_debug_}` is not `{res}`!\n'
+              f'There is an error in how `{type(self).__name__}` '
+              f'values are being passed around.\nRepr: {self!r}')
         self._gcl_cache_debug_ = res
         return res
       self._gcl_cache_ = res
@@ -674,7 +675,8 @@ def __str__(self):
     return (f'<Unevaluated: {self._gcl_construct_}>' if not self._gcl_cache_
             else str(self._gcl_cache_))
   def __repr__(self):
-    return f'<Unevaluated: {self._gcl_construct_}>'
+    return (f'{type(self).__name__}({self._gcl_construct_!r}, '
+            f'cache={self._gcl_cache_!r})')
 
 
 class ModuleToLoad(DeferredValue):
@@ -744,6 +746,10 @@ def _gcl_evaluate_(self, value, ectx):
 
 
 class IfLadderTableIndex(DeferredValue):
+  '''Stashes a sequence of if expressions extracted from an if-else ladder,
+  and evaluates them in order, caching the index of the first truthy expression.
+  Else is represented as -1; the final index in each IfLadderItem table.
+  '''
   def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs)
   def _gcl_explanation_(self):
     return f'Pre-evaluating if-else ladder'
@@ -755,6 +761,9 @@ def _gcl_evaluate_(self, value, ectx):
 
 
 class IfLadderItem(DeferredValue):
+  '''References an extracted IfLadderTableIndex in its final scope to look up a
+  value in a table, generated from its values in an if-else ladder.
+  '''
   def __init__(self, *args, **kwargs):
     super().__init__(*args, **kwargs)
 
@@ -763,10 +772,10 @@ def _gcl_explanation_(self):
 
   def _gcl_evaluate_(self, value, ectx):
     ladder, table = value
-    ladder = ectx.scope._gcl_preprocessors_.get(id(ladder))
-    ectx.Assert(ladder,
-                'Internal error: The preprocessor `!if` directive from which '
-                'this value was assigned was not inherited...')
+    ladder = ectx.scope._gcl_preprocessors_.get(ladder)
+    ectx.Assert(ladder, 'Internal error: The preprocessor '
+        f'`!if` directive from which this value was assigned was not inherited.'
+        f'\nGot: {(*ectx.scope._gcl_preprocessors_.keys(),)}\nWant: {ladder}')
     index = ladder.index._gcl_resolve_(ectx)
     result = table[index]
     while isinstance(result, DeferredValue): result = result._gcl_resolve_(ectx)
@@ -901,7 +910,7 @@ def yamlet_merge(self, other, ectx):
 
 def _UpdateParents(items, parent):
   for i in items:
-    if isinstance(i, (GclDict, DeferredValue, PreprocessingTuple)):
+    if isinstance(i, (GclDict, DeferredValue)):
       i._gcl_update_parent_(parent)
 
 
@@ -984,10 +993,17 @@ class YamletIfStatement(PreprocessingDirective): pass
 class YamletElifStatement(PreprocessingDirective): pass
 class YamletElseStatement(PreprocessingDirective): pass
 class YamletIfElseLadder(PreprocessingDirective):
-  def __init__(self, k, v):
-    self.if_statement = (k, v)
-    self.else_statement = None
-    self.elif_statements = []
+  def __init__(self, k=None, v=None, *, index=None, cond_dvals=None):
+    if index:
+      index._gcl_construct_ = self
+      self.index = index  # An `IfLadderTableIndex` to translate this ladder.
+      self.cond_dvals = cond_dvals
+      return
+    assert isinstance(k, YamletIfStatement)
+    assert isinstance(v, (GclDict, PreprocessingTuple))
+    self.if_statement = (k, v)  # k is the !if expression, v is the GclDict.
+    self.else_statement = None  # Similarly, k is !else None, v is GclDict.
+    self.elif_statements = []   # Sequence of k, v pairs for each !elif.
     self.all_vars = set(v.keys())
 
   def PutElif(self, k, v):
@@ -1011,7 +1027,7 @@ def Finalize(self, filtered_pairs, cErr):
       for k, v in self.else_statement[1]._gcl_noresolve_items_():
         arrays[k][-1] = v
     for k, v in arrays.items():
-      v0 = IfLadderItem((self, v), ladder_point)
+      v0 = IfLadderItem((id(self), v), ladder_point)
       v1 = filtered_pairs.setdefault(k, v0)
       if v0 is not v1:
         filtered_pairs[k] = FlatCompositor([v1, v0], ladder_point, varname=k)
@@ -1020,7 +1036,7 @@ def Finalize(self, filtered_pairs, cErr):
                        for ep in expr_points]
     self.index = IfLadderTableIndex(self, ladder_point)
 
-  def AddPreprocessors(self, preprocessors):
+  def AddToPreprocessorsDict(self, preprocessors):
     preprocessors[id(self)] = self
     preprocessors |= self.if_statement[1]._gcl_preprocessors_
     for elif_statement in self.elif_statements:
@@ -1031,11 +1047,9 @@ def AddPreprocessors(self, preprocessors):
   def _gcl_preprocess_(self, ectx): pass
 
   def yamlet_clone(self, new_scope):
-    other = copy.copy(self)
-    other.cond_dvals = [dv.yamlet_clone(new_scope) for dv in self.cond_dvals]
-    other.index = self.index.yamlet_clone(new_scope)
-    other.index._gcl_construct_ = self
-    return other
+    return YamletIfElseLadder(
+        index=self.index.yamlet_clone(new_scope),
+        cond_dvals=[dv.yamlet_clone(new_scope) for dv in self.cond_dvals])
 
 
 def _OkayToFlatComposite(v1, v2):
@@ -1065,7 +1079,7 @@ def terminateIfDirective():
     nonlocal if_directive, preprocessors
     if if_directive:
       if_directive.Finalize(filtered_pairs, cErr)
-      if_directive.AddPreprocessors(preprocessors)
+      if_directive.AddToPreprocessorsDict(preprocessors)
       if_directive = None
   for k, v in mapping_pairs:
     if isinstance(k, PreprocessingDirective):
@@ -1261,7 +1275,7 @@ def cond(ectx, condition, if_true, if_false):
 }
 
 
-def _GclNameLookup(name, ectx):
+def _GclNameLookup(name, ectx, top=True):
   if name in _BUILTIN_NAMES: return _BUILTIN_NAMES[name](ectx)
   if name in _BUILTIN_VARS: return _BUILTIN_VARS[name]
   if name in ectx.scope:
@@ -1271,7 +1285,16 @@ def _GclNameLookup(name, ectx):
     if get is not null: return get
   if ectx.scope._gcl_parent_:
     with ectx.Scope(ectx.scope._gcl_parent_):
-      return _GclNameLookup(name, ectx)
+      res = _GclNameLookup(name, ectx, top=False)
+      if res is not _undefined: return res
+  if not top: return _undefined
+  sup = ectx.scope._gcl_super_
+  while sup is not None:
+    if sup._gcl_parent_:
+      with ectx.Scope(sup._gcl_parent_):
+        res = _GclNameLookup(name, ectx, top=False)
+        if res is not _undefined: return res
+    sup = sup._gcl_super_
   mvars = ectx.opts.module_vars.get(ectx.ModuleFilename())
   if mvars:
     res = mvars.get(name, _undefined)
@@ -1498,9 +1521,6 @@ def _CompositeGclTuples(tuples, ectx):
   res = None
   for t in tuples:
     if t is None: ectx.Raise(ValueError, 'Expression evaluation failed?')
-    if res:
-      res = res.yamlet_clone(ectx.scope)
-      res.yamlet_merge(t, ectx)
-    else:
-      res = t.yamlet_clone(ectx.scope)
+    if res: res.yamlet_merge(t, ectx)
+    else:   res = t.yamlet_clone(ectx.scope)
   return res