Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit8ddd3b9

Browse files
committed
Generate tokenizer tests from testdata files
1 parent7556f22 commit8ddd3b9

File tree

3 files changed

+85
-51
lines changed

3 files changed

+85
-51
lines changed

‎.pytest.expect‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2,20 +2,20 @@ pytest-expect file v1
22
(2, 7, 11, 'final', 0)
33
b'html5lib/tests/test_encoding.py::test_encoding::[110]': FAIL
44
b'html5lib/tests/test_encoding.py::test_encoding::[111]': FAIL
5-
b'html5lib/tests/test_tokenizer.py::testTokenizer::[4718]': FAIL
6-
b'html5lib/tests/test_tokenizer.py::testTokenizer::[4990]': FAIL
7-
b'html5lib/tests/test_tokenizer.py::testTokenizer::[4993]': FAIL
8-
b'html5lib/tests/test_tokenizer.py::testTokenizer::[4994]': FAIL
9-
b'html5lib/tests/test_tokenizer.py::testTokenizer::[4996]': FAIL
10-
b'html5lib/tests/test_tokenizer.py::testTokenizer::[4997]': FAIL
11-
b'html5lib/tests/test_tokenizer.py::testTokenizer::[4999]': FAIL
12-
b'html5lib/tests/test_tokenizer.py::testTokenizer::[5002]': FAIL
13-
b'html5lib/tests/test_tokenizer.py::testTokenizer::[5003]': FAIL
14-
b'html5lib/tests/test_tokenizer.py::testTokenizer::[5005]': FAIL
15-
b'html5lib/tests/test_tokenizer.py::testTokenizer::[5006]': FAIL
16-
b'html5lib/tests/test_tokenizer.py::testTokenizer::[5008]': FAIL
17-
b'html5lib/tests/test_tokenizer.py::testTokenizer::[5020]': FAIL
18-
b'html5lib/tests/test_tokenizer.py::testTokenizer::[5418]': FAIL
5+
u'html5lib/tests/testdata/tokenizer/test2.test::0::dataState': FAIL
6+
u'html5lib/tests/testdata/tokenizer/test3.test::228::dataState': FAIL
7+
u'html5lib/tests/testdata/tokenizer/test3.test::231::dataState': FAIL
8+
u'html5lib/tests/testdata/tokenizer/test3.test::232::dataState': FAIL
9+
u'html5lib/tests/testdata/tokenizer/test3.test::234::dataState': FAIL
10+
u'html5lib/tests/testdata/tokenizer/test3.test::235::dataState': FAIL
11+
u'html5lib/tests/testdata/tokenizer/test3.test::237::dataState': FAIL
12+
u'html5lib/tests/testdata/tokenizer/test3.test::240::dataState': FAIL
13+
u'html5lib/tests/testdata/tokenizer/test3.test::241::dataState': FAIL
14+
u'html5lib/tests/testdata/tokenizer/test3.test::243::dataState': FAIL
15+
u'html5lib/tests/testdata/tokenizer/test3.test::244::dataState': FAIL
16+
u'html5lib/tests/testdata/tokenizer/test3.test::246::dataState': FAIL
17+
u'html5lib/tests/testdata/tokenizer/test3.test::258::dataState': FAIL
18+
u'html5lib/tests/testdata/tokenizer/test3.test::656::dataState': FAIL
1919
u'html5lib/tests/testdata/tree-construction/foreign-fragment.dat::0::DOM::parser::namespaced': FAIL
2020
u'html5lib/tests/testdata/tree-construction/foreign-fragment.dat::0::DOM::parser::void-namespace': FAIL
2121
u'html5lib/tests/testdata/tree-construction/foreign-fragment.dat::0::ElementTree::parser::namespaced': FAIL

‎html5lib/tests/conftest.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
importos.path
22

33
from .tree_constructionimportTreeConstructionFile
4+
from .tokenizerimportTokenizerFile
45

56
_dir=os.path.abspath(os.path.dirname(__file__))
67
_testdata=os.path.join(_dir,"testdata")
78
_tree_construction=os.path.join(_testdata,"tree-construction")
9+
_tokenizer=os.path.join(_testdata,"tokenizer")
810

911

1012
defpytest_collectstart():
@@ -19,3 +21,6 @@ def pytest_collect_file(path, parent):
1921
return
2022
ifpath.ext==".dat":
2123
returnTreeConstructionFile(path,parent)
24+
elifdir==_tokenizer:
25+
ifpath.ext==".test":
26+
returnTokenizerFile(path,parent)

‎html5lib/tests/test_tokenizer.py‎renamed to ‎html5lib/tests/tokenizer.py‎

Lines changed: 66 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
from __future__importabsolute_import,division,unicode_literals
22

3+
importcodecs
34
importjson
45
importwarnings
56
importre
67

8+
importpytest
79
fromsiximportunichr
810

9-
from .supportimportget_data_files
10-
1111
fromhtml5lib.tokenizerimportHTMLTokenizer
1212
fromhtml5libimportconstants,utils
1313

@@ -172,27 +172,6 @@ def repl(m):
172172
returntest
173173

174174

175-
defrunTokenizerTest(test):
176-
warnings.resetwarnings()
177-
warnings.simplefilter("error")
178-
179-
expected=test['output']
180-
if'lastStartTag'notintest:
181-
test['lastStartTag']=None
182-
parser=TokenizerTestParser(test['initialState'],
183-
test['lastStartTag'])
184-
tokens=parser.parse(test['input'])
185-
received=normalizeTokens(tokens)
186-
errorMsg="\n".join(["\n\nInitial state:",
187-
test['initialState'],
188-
"\nInput:",test['input'],
189-
"\nExpected:",repr(expected),
190-
"\nreceived:",repr(tokens)])
191-
errorMsg=errorMsg
192-
ignoreErrorOrder=test.get('ignoreErrorOrder',False)
193-
asserttokensMatch(expected,received,ignoreErrorOrder,True),errorMsg
194-
195-
196175
def_doCapitalize(match):
197176
returnmatch.group(1).upper()
198177

@@ -205,18 +184,68 @@ def capitalize(s):
205184
returns
206185

207186

208-
deftestTokenizer():
209-
forfilenameinget_data_files('tokenizer','*.test'):
210-
withopen(filename)asfp:
187+
classTokenizerFile(pytest.File):
188+
defcollect(self):
189+
withcodecs.open(str(self.fspath),"r",encoding="utf-8")asfp:
211190
tests=json.load(fp)
212-
if'tests'intests:
213-
forindex,testinenumerate(tests['tests']):
214-
if'initialStates'notintest:
215-
test["initialStates"]= ["Data state"]
216-
if'doubleEscaped'intest:
217-
test=unescape(test)
218-
iftest["input"]isNone:
219-
continue# Not valid input for this platform
220-
forinitialStateintest["initialStates"]:
221-
test["initialState"]=capitalize(initialState)
222-
yieldrunTokenizerTest,test
191+
if'tests'intests:
192+
fori,testinenumerate(tests['tests']):
193+
yieldTokenizerTestCollector(str(i),self,testdata=test)
194+
195+
196+
classTokenizerTestCollector(pytest.Collector):
197+
def__init__(self,name,parent=None,config=None,session=None,testdata=None):
198+
super(TokenizerTestCollector,self).__init__(name,parent,config,session)
199+
if'initialStates'notintestdata:
200+
testdata["initialStates"]= ["Data state"]
201+
if'doubleEscaped'intestdata:
202+
testdata=unescape(testdata)
203+
self.testdata=testdata
204+
205+
defcollect(self):
206+
forinitialStateinself.testdata["initialStates"]:
207+
initialState=capitalize(initialState)
208+
item=TokenizerTest(initialState,
209+
self,
210+
self.testdata,
211+
initialState)
212+
ifself.testdata["input"]isNone:
213+
item.add_marker(pytest.mark.skipif(True,reason="Relies on lone surrogates"))
214+
yielditem
215+
216+
217+
classTokenizerTest(pytest.Item):
218+
def__init__(self,name,parent,test,initialState):
219+
super(TokenizerTest,self).__init__(name,parent)
220+
self.obj=lambda:1# this is to hack around skipif needing a function!
221+
self.test=test
222+
self.initialState=initialState
223+
224+
defruntest(self):
225+
warnings.resetwarnings()
226+
warnings.simplefilter("error")
227+
228+
expected=self.test['output']
229+
if'lastStartTag'notinself.test:
230+
self.test['lastStartTag']=None
231+
parser=TokenizerTestParser(self.initialState,
232+
self.test['lastStartTag'])
233+
tokens=parser.parse(self.test['input'])
234+
received=normalizeTokens(tokens)
235+
errorMsg="\n".join(["\n\nInitial state:",
236+
self.initialState,
237+
"\nInput:",self.test['input'],
238+
"\nExpected:",repr(expected),
239+
"\nreceived:",repr(tokens)])
240+
errorMsg=errorMsg
241+
ignoreErrorOrder=self.test.get('ignoreErrorOrder',False)
242+
asserttokensMatch(expected,received,ignoreErrorOrder,True),errorMsg
243+
244+
defrepr_failure(self,excinfo):
245+
traceback=excinfo.traceback
246+
ntraceback=traceback.cut(path=__file__)
247+
excinfo.traceback=ntraceback.filter()
248+
249+
returnexcinfo.getrepr(funcargs=True,
250+
showlocals=False,
251+
style="short",tbfilter=False)

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp