Port Spitfire to Python 3

scripts/crunner.py

@@ -13,7 +13,10 @@
 import time
 import traceback
 
-import cStringIO as StringIO
+# Add project root to sys.path
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import io as StringIO
 
 from spitfire.compiler import compiler
 from spitfire.compiler import options
@@ -60,11 +63,11 @@ def __getattr__(self, key):
         return self._get_item(key)
 
 
-sys_modules = sys.modules.keys()
+sys_modules = list(sys.modules.keys())
 
 
 def reset_sys_modules():
-    for key in sys.modules.keys():
+    for key in list(sys.modules.keys()):
         if key not in sys_modules:
             del sys.modules[key]
 
@@ -105,18 +108,18 @@ def begin(self):
 
     def end(self):
         self.finish_time = time.time()
-        print >> sys.stderr
+        print(file=sys.stderr)
         if self.num_tests_failed > 0:
             sys.stderr.write(self.buffer.getvalue())
-        print >> sys.stderr, '-' * 70
-        print >> sys.stderr, 'Ran %d tests in %0.3fs' % (
-            self.num_tests_run, self.finish_time - self.start_time)
-        print >> sys.stderr
+        print('-' * 70, file=sys.stderr)
+        print('Ran %d tests in %0.3fs' % (
+            self.num_tests_run, self.finish_time - self.start_time), file=sys.stderr)
+        print(file=sys.stderr)
         if self.num_tests_failed > 0:
-            print >> sys.stderr, 'FAILED (failures=%d)' % self.num_tests_failed
+            print('FAILED (failures=%d)' % self.num_tests_failed, file=sys.stderr)
             sys.exit(1)
         else:
-            print >> sys.stderr, 'OK'
+            print('OK', file=sys.stderr)
             sys.exit(0)
 
     def process_file(self, filename):
@@ -137,31 +140,31 @@ def process_file(self, filename):
                 self.compiler.compile_file(filename)
             except Exception as e:
                 compile_failed = True
-                print >> buffer, '=' * 70
-                print >> buffer, 'FAIL:', modulename, '(' + filename + ')'
-                print >> buffer, '-' * 70
-                traceback.print_exc(None, buffer)
+                print('=' * 70, file=buffer)
+                print('FAIL:', modulename, '(' + filename + ')', file=buffer)
+                print('-' * 70, file=buffer)
+                traceback.print_exc(file=buffer)
             if self.options.debug:
                 if 'parse_tree' in self.options.debug_flags:
-                    print >> buffer, "parse_tree:"
+                    print("parse_tree:", file=buffer)
                     visitor.print_tree(self.compiler._parse_tree, output=buffer)
                 if 'analyzed_tree' in self.options.debug_flags:
-                    print >> buffer, "analyzed_tree:"
+                    print("analyzed_tree:", file=buffer)
                     visitor.print_tree(self.compiler._analyzed_tree,
                                        output=buffer)
                 if 'optimized_tree' in self.options.debug_flags:
-                    print >> buffer, "optimized_tree:"
+                    print("optimized_tree:", file=buffer)
                     visitor.print_tree(self.compiler._optimized_tree,
                                        output=buffer)
                 if 'hoisted_tree' in self.options.debug_flags:
-                    print >> buffer, "hoisted_tree:"
+                    print("hoisted_tree:", file=buffer)
                     visitor.print_tree(self.compiler._hoisted_tree,
                                        output=buffer)
                 if 'source_code' in self.options.debug_flags:
-                    print >> buffer, "source_code:"
+                    print("source_code:", file=buffer)
                     for i, line in enumerate(self.compiler._source_code.split(
                             '\n')):
-                        print >> buffer, '% 3s' % (i + 1), line
+                        print('% 3s' % (i + 1), line, file=buffer)
 
         test_failed = False
         if not self.options.skip_test:
@@ -186,7 +189,7 @@ def process_file(self, filename):
                 try:
                     template_class = getattr(template_module, classname)
                     template = template_class(search_list=self.search_list)
-                    current_output = template.main().encode('utf8')
+                    current_output = template.main()
                 except Exception as e:
                     # An exception here doesn't meant that the test fails
                     # necessarily since libraries don't have a class; as long as
@@ -216,10 +219,10 @@ def process_file(self, filename):
                 if current_output != test_output:
                     test_failed = True
                     if self.options.debug:
-                        print >> buffer, "expected output:"
-                        print >> buffer, test_output
-                        print >> buffer, "actual output:"
-                        print >> buffer, current_output
+                        print("expected output:", file=buffer)
+                        print(test_output, file=buffer)
+                        print("actual output:", file=buffer)
+                        print(current_output, file=buffer)
 
             if compile_failed or test_failed:
                 self.num_tests_failed += 1
@@ -232,23 +235,23 @@ def process_file(self, filename):
                 f = open(current_output_path, 'w')
                 f.write(current_output)
                 f.close()
-                print >> buffer, '=' * 70
-                print >> buffer, 'FAIL:', modulename, '(' + filename + ')'
-                print >> buffer, '-' * 70
-                print >> buffer, 'Compare expected and actual output with:'
-                print >> buffer, ' '.join(['    diff -u', test_output_path,
-                                           current_output_path])
-                print >> buffer, 'Show debug information for the test with:'
+                print('=' * 70, file=buffer)
+                print('FAIL:', modulename, '(' + filename + ')', file=buffer)
+                print('-' * 70, file=buffer)
+                print('Compare expected and actual output with:', file=buffer)
+                print(' '.join(['    diff -u', test_output_path,
+                                           current_output_path]), file=buffer)
+                print('Show debug information for the test with:', file=buffer)
                 test_cmd = [arg for arg in sys.argv if arg not in self.files]
                 if '--debug' not in test_cmd:
                     test_cmd.append('--debug')
                 test_cmd = ' '.join(test_cmd)
-                print >> buffer, '   ', test_cmd, filename
+                print('   ', test_cmd, filename, file=buffer)
                 if raised_exception:
-                    print >> buffer, '-' * 70
-                    print >> buffer, current_output
-                    traceback.print_exc(None, buffer)
-                print >> buffer
+                    print('-' * 70, file=buffer)
+                    print(current_output, file=buffer)
+                    traceback.print_exc(file=buffer)
+                print(file=buffer)
                 self.buffer.write(buffer.getvalue())
             else:
                 if self.options.verbose:
@@ -259,8 +262,6 @@ def process_file(self, filename):
 
 
 if __name__ == '__main__':
-    reload(sys)
-    sys.setdefaultencoding('utf8')
 
     option_parser = optparse.OptionParser()
     options.add_common_options(option_parser)

scripts/spitfire-compile

@@ -20,41 +20,38 @@ def process_file(spt_compiler, filename, options):
 
     def print_output(*args):
         if options.verbose:
-            print >> sys.stderr, ' '.join(args)
+            print(' '.join(args), file=sys.stderr)
 
     try:
         if options.output_file:
             spt_compiler.write_file = False
             if options.output_file == '-':
-                f = sys.stdout
+                f = sys.stdout.buffer
             else:
-                f = open(options.output_file, 'w')
+                f = open(options.output_file, 'wb')
         else:
             spt_compiler.write_file = True
         src_code = spt_compiler.compile_file(filename)
         if options.output_file:
             f.write(src_code)
             f.close()
-    except Exception, e:
+    except Exception as e:
         error_msg = 'Failed processing file: %s' % filename
         if options.verbose:
             logging.exception(error_msg)
         else:
-            print >> sys.stderr, error_msg
-            print >> sys.stderr, e
+            print(error_msg, file=sys.stderr)
+            print(e, file=sys.stderr)
         sys.exit(1)
 
 
 if __name__ == '__main__':
-    reload(sys)
-    sys.setdefaultencoding('utf8')
-
     option_parser = optparse.OptionParser()
     options.add_common_options(option_parser)
     (spt_options, spt_args) = option_parser.parse_args()
 
     if spt_options.version:
-        print >> sys.stderr, 'spitfire %s' % spitfire.__version__
+        print('spitfire %s' % spitfire.__version__, file=sys.stderr)
         sys.exit(0)
 
     spt_compiler_args = compiler.Compiler.args_from_optparse(spt_options)

setup.py

@@ -41,14 +41,15 @@
 
 SCRIPTS = ['scripts/crunner.py', 'scripts/spitfire-compile']
 
-EXT_MODULES = [Extension('spitfire.runtime._baked',
-                         [os.path.join('spitfire', 'runtime', '_baked.c')]),
+EXT_MODULES = [
+    # Extension('spitfire.runtime._baked',
+    #           [os.path.join('spitfire', 'runtime', '_baked.c')]),
                Extension('spitfire.runtime._template',
                          [os.path.join('spitfire', 'runtime', '_template.c')]),
                Extension('spitfire.runtime._udn',
                          [os.path.join('spitfire', 'runtime', '_udn.c')])]
 # Disable C extensions for PyPy.
-if platform.python_implementation() == 'PyPy':
+if platform.python_implementation() == 'PyPy' or os.environ.get('SPITFIRE_SKIP_EXT'):
     EXT_MODULES = None
 
 setup(name=NAME,

spitfire/compiler/analyzer.py

@@ -96,7 +96,7 @@ def build_ast(self, node):
         try:
             if len(ast_node_list) != 1:
                 return ast_node_list
-        except TypeError, e:
+        except TypeError as e:
             self.compiler.error(SemanticAnalyzerError('method: %s, result: %s' %
                                                       (method, ast_node_list)))
 
@@ -446,7 +446,7 @@ def handleMacro(self, pnode, macro_function, parse_rule):
                 fragment_ast = util.parse(macro_output, 'fragment_goal')
             elif isinstance(pnode, ast.CallFunctionNode):
                 fragment_ast = util.parse(macro_output, 'rhs_expression')
-        except Exception, e:
+        except Exception as e:
             self.compiler.error(MacroParseError(e), pos=pnode.pos)
         return self.build_ast(fragment_ast)
 
@@ -464,7 +464,7 @@ def analyzeMacroNode(self, pnode):
         try:
             temp_fragment = util.parse(pnode.value, macro_parse_rule or
                                        'fragment_goal')
-        except Exception, e:
+        except Exception as e:
             self.compiler.error(MacroParseError(e), pos=pnode.pos)
 
         if not self.uses_raw:

spitfire/compiler/ast.py

@@ -3,7 +3,7 @@
 # Use of this source code is governed by a BSD-style
 # license that can be found in the LICENSE file.
 
-import __builtin__
+import builtins
 import copy
 import traceback
 
@@ -61,8 +61,8 @@ def append(self, node):
             else:
                 try:
                     node.parent = self
-                except AttributeError, e:
-                    print e, node
+                except AttributeError as e:
+                    print(e, node)
                     raise
                 self.child_nodes.append(node)
 
@@ -898,7 +898,7 @@ class FragmentNode(ASTNode):
 
 
 class TemplateNode(ASTNode):
-    __builtin_set = frozenset(dir(__builtin__))
+    __builtin_set = frozenset(dir(builtins))
 
     def __init__(self, classname=None, pos=None, **kargs):
         ASTNode.__init__(self, pos=pos, **kargs)
@@ -1033,21 +1033,17 @@ def __delitem__(self, key):
         self._order.remove(key)
         del self._dict[key]
 
-    def keys(self):
-        return list(self.iterkeys())
 
-    def items(self):
-        return list(self.iteritems())
 
-    def iterkeys(self):
+    def keys(self):
         return iter(self._order)
 
-    def iteritems(self):
+    def items(self):
         for key in self._order:
             yield key, self._dict[key]
 
     def update(self, ordered_dict):
-        for key, value in ordered_dict.iteritems():
+        for key, value in ordered_dict.items():
             self[key] = value
 
     def __str__(self):

spitfire/compiler/codegen.py

@@ -5,7 +5,7 @@
 
 import logging
 
-import cStringIO as StringIO
+import io as StringIO
 
 from spitfire.compiler import ast
 
@@ -70,7 +70,7 @@ def get_code(self):
     def generate_python(self, code_node):
         try:
             return code_node.src_line
-        except AttributeError, e:
+        except AttributeError as e:
             self.compiler.error(CodegenError("can't write code_node: %s\n\t%s" %
                                              (code_node, e)))
 
@@ -316,7 +316,7 @@ def codegenASTTargetListNode(self, node):
 
     def codegenASTLiteralNode(self, node):
         if (self.options and not self.options.generate_unicode and
-                isinstance(node.value, basestring)):
+                isinstance(node.value, str)):
             # If the node is the empty string, we should mark it as sanitized by
             # default. Eventually, all string literals should be marked as
             # sanitized.
@@ -694,7 +694,7 @@ def codegenDefault(self, node):
         try:
             return [CodeNode(line % vars(node))
                     for line in v['AST%s_tmpl' % node.__class__.__name__]]
-        except KeyError, e:
+        except KeyError as e:
             self.compiler.error(CodegenError("no codegen for %s %s" % (type(
                 node), vars(node))))
 

spitfire/compiler/compiler.py

@@ -120,7 +120,7 @@ def __init__(self, **kargs):
         self._hoisted_tree = None
         self._source_code = None
 
-        for key, value in kargs.iteritems():
+        for key, value in kargs.items():
             setattr(self, key, value)
 
         if self.analyzer_options is None:
@@ -296,7 +296,7 @@ def write_src_file(self, src_code):
 
         outfile_path = os.path.join(self.output_directory, relative_dir,
                                     outfile_name)
-        outfile = open(outfile_path, 'w')
+        outfile = open(outfile_path, 'wb')
         outfile.write(src_code)
         outfile.close()
 

spitfire/compiler/macros/i18n.py

@@ -5,7 +5,7 @@
 
 import sys
 
-import cStringIO as StringIO
+import io as StringIO
 
 from spitfire.compiler import analyzer
 from spitfire.compiler import ast