integrate first version of "import_library" functionality.

spitfire/compiler/analyzer.py

@@ -189,6 +189,14 @@ def analyzeTemplateNode(self, pnode):
     self.template = pnode.copy(copy_children=False)
     self.template.classname = self.classname
 
+    # Need to build a full list of template_methods before analyzing so we can
+    # modify CallFunctionNodes as we walk the tree below.
+    for child_node in tree_walker(pnode):
+      if isinstance(child_node, DefNode) and not isinstance(child_node, MacroNode):
+        if child_node.name in self.template.template_methods:
+          raise SemanticAnalyzerError('Redefining #def/#block %s' % child_node.name)
+        self.template.template_methods.add(child_node.name)
+
     for pn in self.optimize_parsed_nodes(pnode.child_nodes):
       built_nodes = self.build_ast(pn)
       if built_nodes:
@@ -197,6 +205,9 @@ def analyzeTemplateNode(self, pnode):
     self.template.main_function.child_nodes = self.optimize_buffer_writes(
       self.template.main_function.child_nodes)
 
+    if self.template.extends_nodes and self.template.library:
+      raise SemanticAnalyzerError("library template can't have extends.")
+
     return [self.template]
 
   def analyzeForNode(self, pnode):
@@ -283,7 +294,13 @@ def analyzeImplementsNode(self, pnode):
     return []
 
   def analyzeImportNode(self, pnode):
-    node = ImportNode([self.build_ast(n)[0] for n in pnode.module_name_list])
+    node = ImportNode([self.build_ast(n)[0] for n in pnode.module_name_list], library=pnode.library)
+    if node.library:
+      self.template.library_identifiers.add('.'.join(node.name for node in node.module_name_list))
+      base_extends_identifiers = self.get_base_extends_identifiers()
+      if base_extends_identifiers:
+        node.module_name_list[0:0] = base_extends_identifiers
+
     if node not in self.template.import_nodes:
       self.template.import_nodes.append(node)
     return []
@@ -294,17 +311,10 @@ def analyzeExtendsNode(self, pnode):
     # anything else
     import_node = ImportNode(pnode.module_name_list[:])
     extends_node = ExtendsNode(pnode.module_name_list[:])
-    if (type(pnode) != AbsoluteExtendsNode and
-        self.compiler.base_extends_package):
-      # this means that extends are supposed to all happen relative to some
-      # other package - this is handy for assuring all templates reference
-      # within a tree, say for localization, where each local might have its
-      # own package
-      package_pieces = [IdentifierNode(module_name) for module_name in
-                        self.compiler.base_extends_package.split('.')]
-      import_node.module_name_list[0:0] = package_pieces
-      extends_node.module_name_list[0:0] = package_pieces
-
+    base_extends_identifiers = self.get_base_extends_identifiers()
+    if (type(pnode) != AbsoluteExtendsNode and base_extends_identifiers):
+      import_node.module_name_list[0:0] = base_extends_identifiers
+      extends_node.module_name_list[0:0] = base_extends_identifiers
 
     self.analyzeImportNode(import_node)
 
@@ -317,6 +327,11 @@ def analyzeExtendsNode(self, pnode):
   analyzeAbsoluteExtendsNode = analyzeExtendsNode
 
   def analyzeFromNode(self, pnode):
+    if pnode.library:
+      self.template.library_identifiers.add(pnode.identifier.name)
+      base_extends_identifiers = self.get_base_extends_identifiers()
+      if base_extends_identifiers:
+        pnode.module_name_list[0:0] = base_extends_identifiers
     if pnode not in self.template.from_nodes:
       self.template.from_nodes.append(pnode)
     return []
@@ -343,17 +358,13 @@ def analyzeDefNode(self, pnode):
     #if not isinstance(pnode.parent, TemplateNode):
     #  raise SemanticAnalyzerError("Nested #def or #block directives are not allowed")
 
-    if pnode.name in self.template.template_methods:
-      raise SemanticAnalyzerError('Redefining #def/#block %s' % pnode.name)
-
-    self.template.template_methods.add(pnode.name)
     function = FunctionNode(pnode.name)
     if pnode.parameter_list:
       function.parameter_list = self.build_ast(pnode.parameter_list)[0]
 
     function.parameter_list.child_nodes.insert(0,
                                                ParameterNode(name='self'))
-    
+
     for pn in self.optimize_parsed_nodes(pnode.child_nodes):
       function.extend(self.build_ast(pn))
     function = self.build_ast(function)[0]
@@ -465,6 +476,10 @@ def analyzePlaceholderSubstitutionNode(self, pnode):
           cache_forever = getattr(ph_function, 'cache_forever', False)
           never_cache = getattr(ph_function, 'never_cache', False)
 
+      elif ph_expression.library_function:
+        # Don't escape function calls into library templates.
+        skip_filter = True
+
     if (self.compiler.enable_filters and
         format_string == default_format_string and
         not isinstance(ph_expression, LiteralNode)):
@@ -526,11 +541,35 @@ def analyzeCommentNode(self, pnode):
 
   def analyzeCallFunctionNode(self, pnode):
     fn = pnode
+
+    # The fully qualified library function name iff we figure out
+    # that this is calling into a library.
+    library_function = None
+
     if isinstance(fn.expression, PlaceholderNode):
       macro_handler_name = 'macro_function_%s' % fn.expression.name
       macro_function = self.compiler.macro_registry.get(macro_handler_name)
       if macro_function:
         return self.handleMacro(fn, macro_function)
+      elif self.template.library and fn.expression.name in self.template.template_methods:
+        # Calling another library function from this library function.
+        library_function = fn.expression.name
+
+    elif isinstance(fn.expression, GetUDNNode):
+      module_identifier = [node.name for node in fn.expression.getChildNodes()]
+      module_identifier = '.'.join(module_identifier)
+      if module_identifier in self.template.library_identifiers:
+        # Calling library functions from other templates.
+        library_function = '%s.%s' % (module_identifier, fn.expression.name)
+
+    if library_function:
+      # Replace the placeholder node or UDN resolution with a direct reference
+      # to the library function, either in another imported module or here.
+      fn.expression = IdentifierNode(library_function)
+      # Pass the current template instance into the library function.
+      fn.arg_list.child_nodes.insert(0, IdentifierNode('self'))
+      fn.library_function = True
+
     fn.expression = self.build_ast(fn.expression)[0]
     fn.arg_list = self.build_ast(fn.arg_list)[0]
     return [fn]
@@ -542,6 +581,17 @@ def analyzeFilterNode(self, pnode):
     fn.expression = self.build_ast(fn.expression)[0]
     return [fn]
 
+  def get_base_extends_identifiers(self):
+    if not self.compiler.base_extends_package:
+      return None
+
+    # this means that extends are supposed to all happen relative to some
+    # other package - this is handy for assuring all templates reference
+    # within a tree, say for localization, where each locale might have its
+    # own package
+    return [IdentifierNode(module_name) for module_name in
+                      self.compiler.base_extends_package.split('.')]
+
   # go over the parsed nodes and weed out the parts we don't need
   # it's easier to do this before we morph the AST to look more like python
   def optimize_parsed_nodes(self, node_list):

spitfire/compiler/ast.py

@@ -218,6 +218,8 @@ class CallFunctionNode(ASTNode):
   def __init__(self, expression=None, arg_list=None):
     ASTNode.__init__(self)
     self.expression = expression
+    # Whether we're calling into a library template.
+    self.library_function = False
     if arg_list:
       self.arg_list = arg_list
     else:
@@ -231,6 +233,7 @@ def replace(self, node, replacement_node):
 
   def __eq__(self, node):
     return bool(type(self) == type(node) and
+                self.library_function == node.library_function and
                 self.expression == node.expression and
                 self.arg_list == node.arg_list and
                 self.child_nodes == node.child_nodes)
@@ -486,15 +489,17 @@ class ImplementsNode(ASTNode):
   pass
 
 class ImportNode(ASTNode):
-  def __init__(self, module_name_list):
+  def __init__(self, module_name_list, library=False):
     ASTNode.__init__(self)
     self.module_name_list = module_name_list
+    self.library = library
     # in case you have a different target, save a copy of the
     # orginal name to use for dependency analysis
     self.source_module_name_list = module_name_list[:]
 
   def __eq__(self, node):
     return bool(type(self) == type(node) and
+                self.library == node.library and
                 self.module_name_list == node.module_name_list)
 
   def __hash__(self):
@@ -513,8 +518,8 @@ class AbsoluteExtendsNode(ExtendsNode):
   pass
 
 class FromNode(ImportNode):
-  def __init__(self, module_name_list, identifier, alias=None):
-    ImportNode.__init__(self, module_name_list)
+  def __init__(self, module_name_list, identifier, alias=None, library=False):
+    ImportNode.__init__(self, module_name_list, library=library)
     self.identifier = identifier
     self.alias = alias
 
@@ -686,6 +691,7 @@ def __init__(self, classname=None, **kargs):
     self.global_identifiers = set()
     self.cached_identifiers = set()
     self.template_methods = set()
+    self.library_identifiers = set()
 
   def __str__(self):
     return '%s\nimport:%s\nfrom:%s\nextends:%s\nmain:%s' % (

spitfire/compiler/codegen.py

@@ -88,6 +88,7 @@ def codegenASTTemplateNode(self, node):
       for n in node.from_nodes:
         module_code.extend(self.build_code(n))
       module_code.append_line('')
+
     extends = []
     for n in node.extends_nodes:
       extends.append(self.generate_python(self.build_code(n)[0]))
@@ -106,28 +107,36 @@ def codegenASTTemplateNode(self, node):
     else:
       module_code.append_line('from spitfire.runtime.udn import resolve_placeholder')
       module_code.append_line('from spitfire.runtime.udn import resolve_udn')
+
     module_code.append_line('from spitfire.runtime.template import template_method')
     module_code.append_line('')
+
     if node.cached_identifiers:
       module_code.append_line('# cached identifiers')
       for cached_ph in node.cached_identifiers:
         module_code.append_line('%s = None' % cached_ph.name)
       module_code.append_line('')
-
-    class_code = CodeNode(
-      'class %(classname)s(%(extends_clause)s):' % vars())
-    module_code.append(class_code)
-    for n in node.attr_nodes:
-      class_code.extend(self.build_code(n))
-      class_code.append_line('')
-
+
+    if not node.library:
+      class_code = CodeNode(
+        'class %(classname)s(%(extends_clause)s):' % vars())
+      module_code.append(class_code)
+      for n in node.attr_nodes:
+        class_code.extend(self.build_code(n))
+        class_code.append_line('')
+      def_parent_code = class_code
+    else:
+      # Library functions are written to the module directly.
+      module_code.append_line('')
+      def_parent_code = module_code
+
     for n in node.child_nodes:
-      class_code.extend(self.build_code(n))
-      class_code.append_line('')
+      def_parent_code.extend(self.build_code(n))
+      def_parent_code.append_line('')
 
     # if we aren't extending a template, build out the main function
-    if (not node.extends_nodes and not node.library) or node.implements:
-      class_code.extend(self.build_code(node.main_function))
+    if not node.library and (not node.extends_nodes or node.implements):
+      def_parent_code.extend(self.build_code(node.main_function))
 
     # NOTE(msolo): originally, i thought this would be helpful in case a bit of
     # human error - however, a more robust check is necessary here to make the
@@ -137,7 +146,9 @@ def codegenASTTemplateNode(self, node):
     #   logging.warning("throwing away defined main function because it is not a base class %s %s", self.ast_root.source_path)
     #   logging.warning("%s", flatten_tree(node.main_function))
 
-    if self.options and self.options.enable_psyco:
+    # Don't enable psyco for libraries since there is no class. We might want
+    # to iterate over the library functions and enable it for them instead.
+    if not node.library and self.options and self.options.enable_psyco:
       module_code.append_line('spitfire.runtime.template.enable_psyco(%(classname)s)' % vars())
 
     module_code.append_line(run_tmpl % vars(node))

spitfire/compiler/parser.g

@@ -111,9 +111,9 @@ parser _SpitfireParser:
         |
         'absolute_extends' SPACE modulename CLOSE_DIRECTIVE {{ return AbsoluteExtendsNode(modulename) }}
         |
-        'from' SPACE modulename SPACE 'import' SPACE identifier CLOSE_DIRECTIVE {{ return FromNode(modulename, identifier) }}
+        'from' SPACE modulename SPACE importkeyword SPACE identifier CLOSE_DIRECTIVE {{ return FromNode(modulename, identifier, library=importkeyword) }}
         |
-        'import' SPACE modulename CLOSE_DIRECTIVE {{ return ImportNode(modulename) }}
+        importkeyword SPACE modulename CLOSE_DIRECTIVE {{ return ImportNode(modulename, library=importkeyword) }}
         |
         'slurp' CLOSE_DIRECTIVE {{ return CommentNode('slurp') }}
         |
@@ -132,13 +132,21 @@ parser _SpitfireParser:
         CLOSE_DIRECTIVE {{ return AssignNode(_lhs, _rhs) }}
         |
         'echo' SPACE literal {{ _true_exp = literal }}
-         {{ _test_exp, _false_exp = None, None }}
-         [ SPACE 'if' SPACE expression {{ _test_exp = expression }}
-           [ SPACE 'else' SPACE literal {{ _false_exp = literal }}
-           ]
-         ]
-         CLOSE_DIRECTIVE {{ return EchoNode(_true_exp, _test_exp, _false_exp) }}
-
+        {{ _test_exp, _false_exp = None, None }}
+        [ SPACE 'if' SPACE expression {{ _test_exp = expression }}
+          [ SPACE 'else' SPACE literal {{ _false_exp = literal }}
+          ]
+        ]
+        CLOSE_DIRECTIVE {{ return EchoNode(_true_exp, _test_exp, _false_exp) }}
+
+  rule importkeyword:
+    'import'
+    {{ _library = False }} [
+      '_library'
+      {{ _library = True }}
+    ]
+    {{ return _library }}
+
   rule modulename:
     identifier {{ _module_name_list = [identifier] }}
     ( DOT identifier {{ _module_name_list.append(identifier) }} ) *

spitfire/compiler/parser.py

@@ -31,6 +31,8 @@ class _SpitfireParserScanner(Scanner):
         ("'def'", re.compile('def')),
         ("'i18n'", re.compile('i18n')),
         ("'block'", re.compile('block')),
+        ("'_library'", re.compile('_library')),
+        ("'import'", re.compile('import')),
         ("'else'", re.compile('else')),
         ("'if'", re.compile('if')),
         ("'echo'", re.compile('echo')),
@@ -40,7 +42,6 @@ class _SpitfireParserScanner(Scanner):
         ("'continue'", re.compile('continue')),
         ("'break'", re.compile('break')),
         ("'slurp'", re.compile('slurp')),
-        ("'import'", re.compile('import')),
         ("'from'", re.compile('from')),
         ("'absolute_extends'", re.compile('absolute_extends')),
         ("'extends'", re.compile('extends')),
@@ -134,7 +135,7 @@ def i18n_goal(self):
         return fragment
 
     def statement(self):
-        _token_ = self._peek("'implements'", "'extends'", "'absolute_extends'", "'from'", "'import'", "'slurp'", "'break'", "'continue'", "'attr'", "'filter'", "'set'", "'echo'")
+        _token_ = self._peek("'implements'", "'extends'", "'absolute_extends'", "'from'", "'slurp'", "'break'", "'continue'", "'attr'", "'filter'", "'set'", "'echo'", "'import'")
         if _token_ == "'implements'":
             self._scan("'implements'")
             SPACE = self._scan('SPACE')
@@ -158,17 +159,17 @@ def statement(self):
             SPACE = self._scan('SPACE')
             modulename = self.modulename()
             SPACE = self._scan('SPACE')
-            self._scan("'import'")
+            importkeyword = self.importkeyword()
             SPACE = self._scan('SPACE')
             identifier = self.identifier()
             CLOSE_DIRECTIVE = self.CLOSE_DIRECTIVE()
-            return FromNode(modulename, identifier)
+            return FromNode(modulename, identifier, library=importkeyword)
         elif _token_ == "'import'":
-            self._scan("'import'")
+            importkeyword = self.importkeyword()
             SPACE = self._scan('SPACE')
             modulename = self.modulename()
             CLOSE_DIRECTIVE = self.CLOSE_DIRECTIVE()
-            return ImportNode(modulename)
+            return ImportNode(modulename, library=importkeyword)
         elif _token_ == "'slurp'":
             self._scan("'slurp'")
             CLOSE_DIRECTIVE = self.CLOSE_DIRECTIVE()
@@ -232,6 +233,14 @@ def statement(self):
             CLOSE_DIRECTIVE = self.CLOSE_DIRECTIVE()
             return EchoNode(_true_exp, _test_exp, _false_exp)
 
+    def importkeyword(self):
+        self._scan("'import'")
+        _library = False
+        if self._peek("'_library'", 'SPACE') == "'_library'":
+            self._scan("'_library'")
+            _library = True
+        return _library
+
     def modulename(self):
         identifier = self.identifier()
         _module_name_list = [identifier]
@@ -244,7 +253,7 @@ def modulename(self):
     def directive(self):
         START_DIRECTIVE = self._scan('START_DIRECTIVE')
         _node_list = NodeList()
-        _token_ = self._peek('SINGLE_LINE_COMMENT', 'MULTI_LINE_COMMENT', "'block'", "'i18n'", "'def'", "'for[ \\t]*'", "'strip_lines'", "'if'", "'implements'", "'extends'", "'absolute_extends'", "'from'", "'import'", "'slurp'", "'break'", "'continue'", "'attr'", "'filter'", "'set'", "'echo'", 'END', 'LITERAL_DOLLAR_SIGN', 'LITERAL_BACKSLASH', 'START_DIRECTIVE', 'SPACE', 'NEWLINE', 'START_PLACEHOLDER', 'END_DIRECTIVE', "'#elif'", 'TEXT', "'#else'")
+        _token_ = self._peek('SINGLE_LINE_COMMENT', 'MULTI_LINE_COMMENT', "'block'", "'i18n'", "'def'", "'for[ \\t]*'", "'strip_lines'", "'if'", "'implements'", "'extends'", "'absolute_extends'", "'from'", "'slurp'", "'break'", "'continue'", "'attr'", "'filter'", "'set'", "'echo'", "'import'", 'END', 'LITERAL_DOLLAR_SIGN', 'LITERAL_BACKSLASH', 'START_DIRECTIVE', 'SPACE', 'NEWLINE', 'START_PLACEHOLDER', 'END_DIRECTIVE', "'#elif'", 'TEXT', "'#else'")
         if _token_ == 'SINGLE_LINE_COMMENT':
             SINGLE_LINE_COMMENT = self._scan('SINGLE_LINE_COMMENT')
             _node_list.append(CommentNode(START_DIRECTIVE + SINGLE_LINE_COMMENT))