May 21, 2023 · Apr 19, 2023 · Apr 27, 2023 · Apr 29, 2023 · May 2, 2023 · May 2, 2023
diff --git a/Lib/tabnanny.py b/Lib/tabnanny.py
        errprint("%r: Token Error: %s" % (file, msg))
        return

    except SyntaxError as msg:
        errprint("%r: Token Error: %s" % (file, msg))
        return

    except IndentationError as msg:
        errprint("%r: Indentation Error: %s" % (file, msg))
        return
    return prefix + " " + ', '.join(firsts)

 def process_tokens(tokens):
    try:
        _process_tokens(tokens)
    except TabError as e:
        raise NannyNag(e.lineno, e.msg, e.text)

 def _process_tokens(tokens):
    INDENT = tokenize.INDENT
    DEDENT = tokenize.DEDENT
    NEWLINE = tokenize.NEWLINE
diff --git a/Lib/test/test_tabnanny.py b/Lib/test/test_tabnanny.py
        with TemporaryPyFile(SOURCE_CODES["nannynag_errored"]) as file_path:
            out = f"{file_path!r}: *** Line 3: trouble in tab city! ***\n"
            out += "offending line: '\\tprint(\"world\")\\n'\n"
            out += "indent not equal e.g. at tab size 1\n"
            out += "inconsistent use of tabs and spaces in indentation\n"

            tabnanny.verbose = 1
            self.verify_tabnanny_check(file_path, out=out)
    def test_with_errored_file(self):
        """Should displays error when errored python file is given."""
        with TemporaryPyFile(SOURCE_CODES["wrong_indented"]) as file_path:
            stderr  = f"{file_path!r}:Indentation Error: "
            stderr  = f"{file_path!r}:Token Error: "
            stderr += ('unindent does not match any outer indentation level'
                    ' (<tokenize>, line 3)')
            self.validate_cmd(file_path, stderr=stderr, expect_failure=True)
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
                tokens = list(tokenize(f.readline))
        else:
            filename = "<stdin>"
            tokens = _tokenize(sys.stdin.readline, None)
            tokens = _tokenize(
                (x.encode('utf-8') for x in iter(sys.stdin.readline, "")
            ), "utf-8")


        # Output the tokenization
        for token in tokens:
diff --git a/Lib/trace.py b/Lib/trace.py
    # Add this special case so that the test in the loop passes.
    prev_ttype = token.INDENT
    with open(filename, encoding=encoding) as f:
        print(filename)
        tok = tokenize.generate_tokens(f.readline)
        for ttype, tstr, start, end, line in tok:
            if ttype == token.STRING:
diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c
            }
            return -1;
        case E_DEDENT:
            PyErr_SetString(PyExc_IndentationError,
                            "unindent does not match any outer indentation level");
            PyErr_Format(PyExc_IndentationError,
                        "unindent does not match any outer indentation level "
                        "(<tokenize>, line %d)",
                        tok->lineno);
            return -1;
        case E_INTR:
            if (!PyErr_Occurred()) {
        default:
            msg = "unknown tokenization error";
    }
    PyErr_SetString(errtype, msg);

    // TODO: Clean up this code and factor out common error paths

    PyObject* errstr = NULL;
    PyObject* error_line = NULL;

    Py_ssize_t size = tok->inp - tok->buf;
    error_line = PyUnicode_DecodeUTF8(tok->buf, size, "replace");
    if (!error_line) {
        goto error;
    }
    PyObject *tmp = Py_BuildValue("(OnnOii)", tok->filename, tok->lineno, 0, error_line, 0, 0);
    if (!tmp) {
        goto error;
    }
    Py_CLEAR(error_line);
    errstr = PyUnicode_FromString(msg);
    if (!errstr) {
        goto error;
    }
    PyObject* value = PyTuple_Pack(2, errstr, tmp);
    Py_DECREF(errstr);
    Py_DECREF(tmp);
    if (!value) {
        goto error;
    }
    PyErr_SetObject(errtype, value);
    Py_DECREF(value);
    return 0;
 error:
    Py_XDECREF(errstr);
    Py_XDECREF(error_line);
    return -1;
 }
Original file line number	Diff line number	Diff line change
Expand Up		@@ -107,6 +107,10 @@ def check(file):
		errprint("%r: Token Error: %s" % (file, msg))
		return

		except SyntaxError as msg:
		errprint("%r: Token Error: %s" % (file, msg))
		return

		except IndentationError as msg:
		errprint("%r: Indentation Error: %s" % (file, msg))
		return
Expand DownExpand Up		@@ -272,6 +276,12 @@ def format_witnesses(w):
		return prefix + " " + ', '.join(firsts)

		def process_tokens(tokens):
		try:
		_process_tokens(tokens)
		except TabError as e:
		raise NannyNag(e.lineno, e.msg, e.text)

		def _process_tokens(tokens):
		INDENT = tokenize.INDENT
		DEDENT = tokenize.DEDENT
		NEWLINE = tokenize.NEWLINE
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -223,7 +223,7 @@ def test_when_nannynag_error_verbose(self):
		with TemporaryPyFile(SOURCE_CODES["nannynag_errored"]) as file_path:
		out = f"{file_path!r}: * Line 3: trouble in tab city! *\n"
		out += "offending line: '\\tprint(\"world\")\\n'\n"
		out += "indent not equal e.g. at tab size 1\n"
		out += "inconsistent use of tabs and spaces in indentation\n"

		tabnanny.verbose = 1
		self.verify_tabnanny_check(file_path, out=out)
Expand DownExpand Up		@@ -315,7 +315,7 @@ def validate_cmd(self, *args, stdout="", stderr="", partial=False, expect_failur
		def test_with_errored_file(self):
		"""Should displays error when errored python file is given."""
		with TemporaryPyFile(SOURCE_CODES["wrong_indented"]) as file_path:
		stderr = f"{file_path!r}:Indentation Error: "
		stderr = f"{file_path!r}:Token Error: "
		stderr += ('unindent does not match any outer indentation level'
		' (<tokenize>, line 3)')
		self.validate_cmd(file_path, stderr=stderr, expect_failure=True)
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -517,7 +517,10 @@ def error(message, filename=None, location=None):
		tokens = list(tokenize(f.readline))
		else:
		filename = "<stdin>"
		tokens = _tokenize(sys.stdin.readline, None)
		tokens = _tokenize(
		(x.encode('utf-8') for x in iter(sys.stdin.readline, "")
		), "utf-8")


		# Output the tokenization
		for token in tokens:
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -360,7 +360,6 @@ def _find_strings(filename, encoding=None):
		# Add this special case so that the test in the loop passes.
		prev_ttype = token.INDENT
		with open(filename, encoding=encoding) as f:
		print(filename)
		tok = tokenize.generate_tokens(f.readline)
		for ttype, tstr, start, end, line in tok:
		if ttype == token.STRING:
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -89,8 +89,10 @@ _tokenizer_error(struct tok_state *tok)
		}
		return -1;
		case E_DEDENT:
		PyErr_SetString(PyExc_IndentationError,
		"unindent does not match any outer indentation level");
		PyErr_Format(PyExc_IndentationError,
		"unindent does not match any outer indentation level "
		"(<tokenize>, line %d)",
		tok->lineno);
		return -1;
		case E_INTR:
		if (!PyErr_Occurred()) {
Expand All		@@ -115,7 +117,38 @@ _tokenizer_error(struct tok_state *tok)
		default:
		msg = "unknown tokenization error";
		}
		PyErr_SetString(errtype, msg);

		// TODO: Clean up this code and factor out common error paths

		PyObject* errstr = NULL;
		PyObject* error_line = NULL;

		Py_ssize_t size = tok->inp - tok->buf;
		error_line = PyUnicode_DecodeUTF8(tok->buf, size, "replace");
		if (!error_line) {
		goto error;
		}
		PyObject *tmp = Py_BuildValue("(OnnOii)", tok->filename, tok->lineno, 0, error_line, 0, 0);
		if (!tmp) {
		goto error;
		}
		Py_CLEAR(error_line);
		errstr = PyUnicode_FromString(msg);
		if (!errstr) {
		goto error;
		}
		PyObject* value = PyTuple_Pack(2, errstr, tmp);
		Py_DECREF(errstr);
		Py_DECREF(tmp);
		if (!value) {
		goto error;
		}
		PyErr_SetObject(errtype, value);
		Py_DECREF(value);
		return 0;
		error:
		Py_XDECREF(errstr);
		Py_XDECREF(error_line);
		return -1;
		}

Expand Down