Store expected failures and move to py.test

.travis.yml

@@ -7,6 +7,8 @@ python:
   - "3.4"
   - "3.5"
   - "pypy"
+  - "pypy3"
+
 sudo: false
 
 cache:
@@ -36,7 +38,7 @@ install:
   - bash requirements-install.sh
 
 script:
-  - nosetests
+  - py.test
   - bash flake8-run.sh
 
 after_script:

CHANGES.rst

@@ -7,9 +7,14 @@ Change Log
 Released on XXX
 
 * Added ordereddict as a mandatory dependency on Python 2.6.
+
 * Added ``lxml``, ``genshi``, ``datrie``, ``charade``, and ``all`` extras that
   will do the right thing based on the specific interpreter implementation.
 
+* Now requires the ``mock`` package for the testsuite.
+
+* Cease supporting DATrie under PyPy.
+
 
 0.9999999/1.0b8
 ~~~~~~~~~~~~~~~

MANIFEST.in

@@ -1,6 +1,10 @@
 include LICENSE
+include AUTHORS.rst
 include CHANGES.rst
 include README.rst
 include requirements*.txt
+include .pytest.expect
+include tox.ini
+include pytest.ini
 graft html5lib/tests/testdata
 recursive-include html5lib/tests *.py

README.rst

@@ -104,8 +104,8 @@ Optional Dependencies
 The following third-party libraries may be used for additional
 functionality:
 
-- ``datrie`` can be used to improve parsing performance (though in
-  almost all cases the improvement is marginal);
+- ``datrie`` can be used under CPython to improve parsing performance
+  (though in almost all cases the improvement is marginal);
 
 - ``lxml`` is supported as a tree format (for both building and
   walking) under CPython (but *not* PyPy where it is known to cause
@@ -132,9 +132,9 @@ Please report any bugs on the `issue tracker
 Tests
 -----
 
-Unit tests require the ``nose`` library and can be run using the
-``nosetests`` command in the root directory; ``ordereddict`` is
-required under Python 2.6. All should pass.
+Unit tests require the ``pytest`` and ``mock`` libraries and can be
+run using the ``py.test`` command in the root directory;
+``ordereddict`` is required under Python 2.6. All should pass.
 
 Test data are contained in a separate `html5lib-tests
 <https://github.com/html5lib/html5lib-tests>`_ repository and included

html5lib/tests/conftest.py

@@ -0,0 +1,21 @@
+import os.path
+
+from .tree_construction import TreeConstructionFile
+
+_dir = os.path.abspath(os.path.dirname(__file__))
+_testdata = os.path.join(_dir, "testdata")
+_tree_construction = os.path.join(_testdata, "tree-construction")
+
+
+def pytest_collectstart():
+    """check to see if the git submodule has been init'd"""
+    pass
+
+
+def pytest_collect_file(path, parent):
+    dir = os.path.abspath(path.dirname)
+    if dir == _tree_construction:
+        if path.basename == "template.dat":
+            return
+        if path.ext == ".dat":
+            return TreeConstructionFile(path, parent)

html5lib/tests/support.py

@@ -21,36 +21,30 @@
 
 # Try whatever etree implementations are available from a list that are
 #"supposed" to work
-try:
-    import xml.etree.ElementTree as ElementTree
-    treeTypes['ElementTree'] = treebuilders.getTreeBuilder("etree", ElementTree, fullTree=True)
-except ImportError:
-    try:
-        import elementtree.ElementTree as ElementTree
-        treeTypes['ElementTree'] = treebuilders.getTreeBuilder("etree", ElementTree, fullTree=True)
-    except ImportError:
-        pass
+import xml.etree.ElementTree as ElementTree
+treeTypes['ElementTree'] = treebuilders.getTreeBuilder("etree", ElementTree, fullTree=True)
 
 try:
     import xml.etree.cElementTree as cElementTree
-    treeTypes['cElementTree'] = treebuilders.getTreeBuilder("etree", cElementTree, fullTree=True)
 except ImportError:
-    try:
-        import cElementTree
+    treeTypes['cElementTree'] = None
+else:
+    # On Python 3.3 and above cElementTree is an alias, don't run them twice.
+    if cElementTree.Element is ElementTree.Element:
+        treeTypes['cElementTree'] = None
+    else:
         treeTypes['cElementTree'] = treebuilders.getTreeBuilder("etree", cElementTree, fullTree=True)
-    except ImportError:
-        pass
 
 try:
     import lxml.etree as lxml  # flake8: noqa
 except ImportError:
-    pass
+    treeTypes['lxml'] = None
 else:
     treeTypes['lxml'] = treebuilders.getTreeBuilder("lxml")
 
 
 def get_data_files(subdirectory, files='*.dat'):
-    return glob.glob(os.path.join(test_dir, subdirectory, files))
+    return sorted(glob.glob(os.path.join(test_dir, subdirectory, files)))
 
 
 class DefaultDict(dict):
@@ -71,9 +65,6 @@ def __init__(self, filename, newTestHeading="data", encoding="utf8"):
         self.encoding = encoding
         self.newTestHeading = newTestHeading
 
-    def __del__(self):
-        self.f.close()
-
     def __iter__(self):
         data = DefaultDict(None)
         key = None
@@ -128,7 +119,7 @@ def convertData(data):
 def errorMessage(input, expected, actual):
     msg = ("Input:\n%s\nExpected:\n%s\nRecieved\n%s\n" %
            (repr(input), repr(expected), repr(actual)))
-    if sys.version_info.major == 2:
+    if sys.version_info[0] == 2:
         msg = msg.encode("ascii", "backslashreplace")
     return msg
 

html5lib/tests/test_meta.py

@@ -0,0 +1,41 @@
+from __future__ import absolute_import, division, unicode_literals
+
+import six
+from mock import Mock
+
+from . import support
+
+
+def _createReprMock(r):
+    """Creates a mock with a __repr__ returning r
+
+    Also provides __str__ mock with default mock behaviour"""
+    mock = Mock()
+    mock.__repr__ = Mock()
+    mock.__repr__.return_value = r
+    mock.__str__ = Mock(wraps=mock.__str__)
+    return mock
+
+
+def test_errorMessage():
+    # Create mock objects to take repr of
+    input = _createReprMock("1")
+    expected = _createReprMock("2")
+    actual = _createReprMock("3")
+
+    # Run the actual test
+    r = support.errorMessage(input, expected, actual)
+
+    # Assertions!
+    if six.PY2:
+        assert b"Input:\n1\nExpected:\n2\nRecieved\n3\n" == r
+    else:
+        assert six.PY3
+        assert "Input:\n1\nExpected:\n2\nRecieved\n3\n" == r
+
+    assert input.__repr__.call_count == 1
+    assert expected.__repr__.call_count == 1
+    assert actual.__repr__.call_count == 1
+    assert not input.__str__.called
+    assert not expected.__str__.called
+    assert not actual.__str__.called

html5lib/tests/test_parser2.py

@@ -40,12 +40,12 @@ def test_namespace_html_elements_1_dom(self):
     def test_namespace_html_elements_0_etree(self):
         parser = html5parser.HTMLParser(namespaceHTMLElements=True)
         doc = parser.parse("<html></html>")
-        self.assertTrue(list(doc)[0].tag == "{%s}html" % (namespaces["html"],))
+        self.assertTrue(doc.tag == "{%s}html" % (namespaces["html"],))
 
     def test_namespace_html_elements_1_etree(self):
         parser = html5parser.HTMLParser(namespaceHTMLElements=False)
         doc = parser.parse("<html></html>")
-        self.assertTrue(list(doc)[0].tag == "html")
+        self.assertTrue(doc.tag == "html")
 
     def test_unicode_file(self):
         parser = html5parser.HTMLParser()

html5lib/tests/test_tokenizer.py

@@ -109,6 +109,7 @@ def tokensMatch(expectedTokens, receivedTokens, ignoreErrorOrder,
                 token.pop()
 
     if not ignoreErrorOrder and not ignoreErrors:
+        expectedTokens = concatenateCharacterTokens(expectedTokens)
         return expectedTokens == receivedTokens
     else:
         # Sort the tokens into two groups; non-parse errors and parse errors
@@ -121,6 +122,7 @@ def tokensMatch(expectedTokens, receivedTokens, ignoreErrorOrder,
                 else:
                     if not ignoreErrors:
                         tokens[tokenType][1].append(token)
+            tokens[tokenType][0] = concatenateCharacterTokens(tokens[tokenType][0])
         return tokens["expected"] == tokens["received"]
 
 
@@ -174,13 +176,12 @@ def runTokenizerTest(test):
     warnings.resetwarnings()
     warnings.simplefilter("error")
 
-    expected = concatenateCharacterTokens(test['output'])
+    expected = test['output']
     if 'lastStartTag' not in test:
         test['lastStartTag'] = None
     parser = TokenizerTestParser(test['initialState'],
                                  test['lastStartTag'])
     tokens = parser.parse(test['input'])
-    tokens = concatenateCharacterTokens(tokens)
     received = normalizeTokens(tokens)
     errorMsg = "\n".join(["\n\nInitial state:",
                           test['initialState'],