Apr 18, 2025 · Apr 29, 2023 · Apr 29, 2023 · Apr 29, 2023 · Apr 30, 2023 · Apr 30, 2023
diff --git a/Include/internal/pycore_bytesobject.h b/Include/internal/pycore_bytesobject.h
    const void *bytes,
    Py_ssize_t size);


 /** Dedent a UTF-8 encoded string.
 * behavior is expected to match `textwrap.dedent`
 *
 * return value:
 * 0, no need to dedent, writer untouched
 * 1, success
 * -1, failure
 *
 * str is the beginning of the string to dedent.
 * expecting (str != NULL)
 *
 * len is the length of the string to dedent.
 * expecting (len >= 0)
 *
 * writer is a _PyBytesWriter object to write the dedented string.
 * expecting (writer != NULL)
 *
 * p points to a char* indicating the current position in the _PyBytesWriter.
 * It is updated to the new position after writing the dedented string on exit.
 * expecting (p != NULL && *p != NULL)
 */
 PyAPI_FUNC(int)
 _PyBytes_Dedent(const char *str, Py_ssize_t len, _PyBytesWriter *writer,
                char **p);
 #ifdef __cplusplus
 }
 #endif
diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-29-23-15-38.gh-issue-103997.BS3uVt.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-29-23-15-38.gh-issue-103997.BS3uVt.rst
 Strings passed to  "-c" are now automatically dedented (common leading whitespace is removed). This allows "python -c" invocations to be indented in shell scripts without causing indentation errors.
 String arguments passed to  "-c" are now automatically dedented as if by
 :func:`textwrap.dedent`. This allows "python -c" invocations to be indented
 in shell scripts without causing indentation errors.

 Add a private API :c:func:`_PyBytes_Dedent`.

 (Patch by Jon Crall and Steven Sun)
diff --git a/Modules/main.c b/Modules/main.c
 #include "pycore_pathconfig.h"    // _PyPathConfig_ComputeSysPath0()
 #include "pycore_pylifecycle.h"   // _Py_PreInitializeFromPyArgv()
 #include "pycore_pystate.h"       // _PyInterpreterState_GET()
 #include "pycore_bytesobject.h"   // _PyBytesWriter, _PyBytes_Dedent()

 /* Includes for exit_sigint() */
 #include <stdio.h>                // perror()
 }

 /* Strip common leading whitespace, just as textwrap.dedent.
   Itsteals 1 reference from bytes if succeeded, else it will return NULL. */
   Itreturns a new reference. */
 static PyObject *
 dedent_utf8_bytes(PyObject *bytes)
 {
    if (bytes == NULL || !PyBytes_CheckExact(bytes)) {
        return NULL;
    }
    assert(bytes == NULL || !PyBytes_CheckExact(bytes->ob_type));

    char *start;
    Py_ssize_t nchars;

    char *start;
    if (PyBytes_AsStringAndSize(bytes, &start, &nchars) != 0) {
        return NULL;
    }

    char *end = start + nchars;
    assert(start < end);

    char *candidate_start = NULL;
    Py_ssize_t candidate_len = 0;

    for (char *iter = start; iter < end; ++iter) {
        char *line_start = iter;
        char *leading_whitespace_end = NULL;

        // scan the whole line
        while (iter < end && *iter != '\n') {
            if (!leading_whitespace_end && *iter != ' ' && *iter != '\t') {
                if (iter == line_start) {
                    // some line has no indent, fast exit!
                    return bytes;
                }
                leading_whitespace_end = iter;
            }
            ++iter;
        }

        // if this line has all white space, skip it
        if (!leading_whitespace_end) {
            continue;
        }

        if (!candidate_start) {
            candidate_start = line_start;
            candidate_len = leading_whitespace_end - line_start;
            assert(candidate_len > 0);
        } else {
            /* We then compare with the current longest leading whitespace.

               [line_start, leading_whitespace_end) is the leading whitespace of
               this line,

               [candidate_start, candidate_start + candidate_len)
               is the leading whitespace of the current longest leading
               whitespace. */
            Py_ssize_t new_candidate_len = 0;

            for (char *candidate_iter = candidate_start,
                      *line_iter = line_start;
                 candidate_iter < candidate_start + candidate_len &&
                 line_iter < leading_whitespace_end;
                 ++candidate_iter, ++line_iter) {
                if (*candidate_iter != *line_iter) {
                    break;
                }
                ++new_candidate_len;
            }

            candidate_len = new_candidate_len;
            if (candidate_len == 0) {
                return bytes;
            }
        }
    }

    if (candidate_len == 0) {
        return bytes;
    }
    assert(candidate_len > 0);

    // trigger a dedent
    char *p;
    PyObject *new_bytes;
    char *line_start;
    Py_ssize_t new_line_len;
    bool in_leading_space;
    _PyBytesWriter writer;

    _PyBytesWriter_Init(&writer);
    p = _PyBytesWriter_Alloc(&writer, nchars);
 char *p = _PyBytesWriter_Alloc(&writer, nchars);
    if (p == NULL) {
 goto error;
 return NULL;
    }

    for (char *iter = start; iter < end; ++iter) {
        line_start = iter;
        in_leading_space = true;

        // iterate over a line
        while (iter < end && *iter != '\n') {
            if (in_leading_space && *iter != ' ' && *iter != '\t') {
                in_leading_space = false;
            }
            ++iter;
        }

        // invariant: *iter == '\n' or iter == end

        // if this line has all white space, write '\n'
        if (in_leading_space) {
            p = _PyBytesWriter_Prepare(&writer, p, 1);
            if (p == NULL) {
                goto error;
            }
            *p++ = '\n';
            continue;
        }

        // copy [new_line_start + candidate_len, iter) to buffer, then append
        // '\n'
        new_line_len = iter - line_start - candidate_len;
        assert(new_line_len >= 0);
        p = _PyBytesWriter_Prepare(&writer, p, new_line_len + 1);
        if (p == NULL) {
            goto error;
        }
        memcpy(p, line_start + candidate_len, new_line_len);

        p += new_line_len;

        // this may always append '\n' at the end of `new_bytes`
        *p++ = '\n';
    int ret = _PyBytes_Dedent(start, nchars, &writer, &p);
    if (ret < 0) {
        return NULL;
    }

 new_bytes = _PyBytesWriter_Finish(&writer, p);
 if (new_bytes == NULL) {
 goto error;
    if (ret == 0) {
    Py_INCREF(bytes);
    _PyBytesWriter_Dealloc(&writer);
 return bytes;
    }
    Py_DECREF(bytes);
    return new_bytes;

 error:
    _PyBytesWriter_Dealloc(&writer);
    return NULL;
    return _PyBytesWriter_Finish(&writer, p);
 }

 static int
        goto error;
    }

    bytes = dedent_utf8_bytes(bytes);
    if (bytes == NULL) {
    PyObject *new_bytes = dedent_utf8_bytes(bytes);
    if (new_bytes == NULL) {
        Py_DECREF(bytes);
        goto error;
    }
    Py_SETREF(bytes, new_bytes);

    PyCompilerFlags cf = _PyCompilerFlags_INIT;
    cf.cf_flags |= PyCF_IGNORE_COOKIE;
diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c
 }


 /* Algorithms on bytes */

 void
 _PyBytes_Repeat(char* dest, Py_ssize_t len_dest,
    const char* src, Py_ssize_t len_src)
    }
 }

 /** Dedent a UTF-8 encoded string.
 * behavior is expected to match `textwrap.dedent`
 *
 * return value:
 * 0, no need to dedent, writer untouched
 * 1, success
 * -1, failure
 *
 * str is the beginning of the string to dedent.
 * expecting (str != NULL)
 *
 * len is the length of the string to dedent.
 * expecting (len >= 0)
 *
 * writer is a _PyBytesWriter object to write the dedented string.
 * expecting (writer != NULL)
 *
 * p points to a char* indicating the current position in the _PyBytesWriter.
 * It is updated to the new position after writing the dedented string on exit.
 * expecting (p != NULL && *p != NULL)
 */
 int
 _PyBytes_Dedent(const char *str, Py_ssize_t len, _PyBytesWriter *writer,
                char **p)
 {
    assert(str);
    assert(p != NULL && *p != NULL);
    assert(writer);

    if (len <= 0)
        return 0;

    const char *end = str + len;
    assert(str < end); // prevent overflow when len is too large

    const char *candidate_start = NULL;
    Py_ssize_t candidate_len = 0;

    for (const char *iter = str; iter < end; ++iter) {
        const char *line_start = iter;
        const char *leading_whitespace_end = NULL;

        // scan the whole line
        while (iter < end && *iter != '\n') {
            if (!leading_whitespace_end && *iter != ' ' && *iter != '\t') {
                if (iter == line_start) {
                    // some line has no indent, fast exit!
                    return 0;
                }
                leading_whitespace_end = iter;
            }
            ++iter;
        }

        // if this line has all white space, skip it
        if (!leading_whitespace_end) {
            continue;
        }

        if (!candidate_start) {
            candidate_start = line_start;
            candidate_len = leading_whitespace_end - line_start;
            assert(candidate_len > 0);
        } else {
            /* We then compare with the current longest leading whitespace.

               [line_start, leading_whitespace_end) is the leading whitespace of
               this line,

               [candidate_start, candidate_start + candidate_len)
               is the leading whitespace of the current longest leading
               whitespace. */
            Py_ssize_t new_candidate_len = 0;

            for (const char *candidate_iter = candidate_start,
                            *line_iter = line_start;
                 candidate_iter < candidate_start + candidate_len &&
                 line_iter < leading_whitespace_end;
                 ++candidate_iter, ++line_iter) {
                if (*candidate_iter != *line_iter) {
                    break;
                }
                ++new_candidate_len;
            }

            candidate_len = new_candidate_len;
            if (candidate_len == 0) {
                return 0;
            }
        }
    }

    assert(candidate_len >= 0);
    if (candidate_len == 0) {
        return 0;
    }

    // trigger a dedent

    // prepare the writer
    char *p_ = _PyBytesWriter_Prepare(writer, *p, len);
    if (p_ == NULL) {
        *p = NULL;
        return -1;
    }

    for (const char *iter = str; iter < end; ++iter) {
        const char *line_start = iter;
        bool in_leading_space = true;

        // iterate over a line to find the end of a line
        while (iter < end && *iter != '\n') {
            if (in_leading_space && *iter != ' ' && *iter != '\t') {
                in_leading_space = false;
            }
            ++iter;
        }

        // invariant: *iter == '\n' or iter == end
        bool append_newline = iter < end;

        // if this line has all white space, write '\n'
        if (in_leading_space && append_newline) {
            *p_++ = '\n';
            continue;
        }

        /* copy [new_line_start + candidate_len, iter) to buffer, then
            conditionally append '\n' */

        Py_ssize_t new_line_len = iter - line_start - candidate_len;
        assert(new_line_len >= 0);

        memcpy(p_, line_start + candidate_len, new_line_len);

        p_ += new_line_len;

        if (append_newline) {
            *p_++ = '\n';
        }
    }

    *p = p_;
    return 1;
 }
Original file line number	Diff line number	Diff line change
Expand Up		@@ -122,6 +122,31 @@ PyAPI_FUNC(void) _PyBytesWriter_WriteBytes(_PyBytesWriter writer,
		const void *bytes,
		Py_ssize_t size);


		/** Dedent a UTF-8 encoded string.
		* behavior is expected to match `textwrap.dedent`
		*
		* return value:
		* 0, no need to dedent, writer untouched
		* 1, success
		* -1, failure
		*
		* str is the beginning of the string to dedent.
		* expecting (str != NULL)
		*
		* len is the length of the string to dedent.
		* expecting (len >= 0)
		*
sunmy2019 marked this conversation as resolved. OutdatedShow resolvedHide resolved
		* writer is a _PyBytesWriter object to write the dedented string.
		* expecting (writer != NULL)
		*
		* p points to a char* indicating the current position in the _PyBytesWriter.
		* It is updated to the new position after writing the dedented string on exit.
		* expecting (p != NULL && *p != NULL)
		*/
		PyAPI_FUNC(int)
		_PyBytes_Dedent(const char str, Py_ssize_t len, _PyBytesWriter writer,
		char **p);
		#ifdef __cplusplus
		}
		#endif
Expand Down
Original file line number	Diff line number	Diff line change
		@@ -1 +1,7 @@
		Strings passed to "-c" are now automatically dedented (common leading whitespace is removed). This allows "python -c" invocations to be indented in shell scripts without causing indentation errors.
		String arguments passed to "-c" are now automatically dedented as if by
		:func:`textwrap.dedent`. This allows "python -c" invocations to be indented
		in shell scripts without causing indentation errors.
sunmy2019 marked this conversation as resolved. OutdatedShow resolvedHide resolved

		Add a private API :c:func:`_PyBytes_Dedent`.
sunmy2019 marked this conversation as resolved. OutdatedShow resolvedHide resolved

		(Patch by Jon Crall and Steven Sun)
Original file line number	Diff line number	Diff line change
Expand Up		@@ -7,6 +7,7 @@
		#include "pycore_pathconfig.h" // _PyPathConfig_ComputeSysPath0()
		#include "pycore_pylifecycle.h" // _Py_PreInitializeFromPyArgv()
		#include "pycore_pystate.h" // _PyInterpreterState_GET()
		#include "pycore_bytesobject.h" // _PyBytesWriter, _PyBytes_Dedent()
sunmy2019 marked this conversation as resolved. OutdatedShow resolvedHide resolved

		/* Includes for exit_sigint() */
		#include <stdio.h> // perror()
Expand DownExpand Up		@@ -229,150 +230,35 @@ pymain_import_readline(const PyConfig *config)
		}

		/* Strip common leading whitespace, just as textwrap.dedent.
		Itsteals 1 reference from bytes if succeeded, else it will return NULL. */
		Itreturns a new reference. */
		static PyObject *
		dedent_utf8_bytes(PyObject *bytes)
		{
		if (bytes == NULL \|\| !PyBytes_CheckExact(bytes)) {
		return NULL;
		}
		assert(bytes == NULL \|\| !PyBytes_CheckExact(bytes->ob_type));

		char *start;
		Py_ssize_t nchars;

		char *start;
		if (PyBytes_AsStringAndSize(bytes, &start, &nchars) != 0) {
		return NULL;
		}

		char *end = start + nchars;
		assert(start < end);

		char *candidate_start = NULL;
		Py_ssize_t candidate_len = 0;

		for (char *iter = start; iter < end; ++iter) {
		char *line_start = iter;
		char *leading_whitespace_end = NULL;

		// scan the whole line
		while (iter < end && *iter != '\n') {
		if (!leading_whitespace_end && iter != ' ' && iter != '\t') {
		if (iter == line_start) {
		// some line has no indent, fast exit!
		return bytes;
		}
		leading_whitespace_end = iter;
		}
		++iter;
		}

		// if this line has all white space, skip it
		if (!leading_whitespace_end) {
		continue;
		}

		if (!candidate_start) {
		candidate_start = line_start;
		candidate_len = leading_whitespace_end - line_start;
		assert(candidate_len > 0);
		} else {
		/* We then compare with the current longest leading whitespace.

		[line_start, leading_whitespace_end) is the leading whitespace of
		this line,

		[candidate_start, candidate_start + candidate_len)
		is the leading whitespace of the current longest leading
		whitespace. */
		Py_ssize_t new_candidate_len = 0;

		for (char *candidate_iter = candidate_start,
		*line_iter = line_start;
		candidate_iter < candidate_start + candidate_len &&
		line_iter < leading_whitespace_end;
		++candidate_iter, ++line_iter) {
		if (candidate_iter != line_iter) {
		break;
		}
		++new_candidate_len;
		}

		candidate_len = new_candidate_len;
		if (candidate_len == 0) {
		return bytes;
		}
		}
		}

		if (candidate_len == 0) {
		return bytes;
		}
		assert(candidate_len > 0);

		// trigger a dedent
		char *p;
		PyObject *new_bytes;
		char *line_start;
		Py_ssize_t new_line_len;
		bool in_leading_space;
		_PyBytesWriter writer;

		_PyBytesWriter_Init(&writer);
		p = _PyBytesWriter_Alloc(&writer, nchars);
		char *p = _PyBytesWriter_Alloc(&writer, nchars);
		if (p == NULL) {
		goto error;
		return NULL;
		}

		for (char *iter = start; iter < end; ++iter) {
		line_start = iter;
		in_leading_space = true;

		// iterate over a line
		while (iter < end && *iter != '\n') {
		if (in_leading_space && iter != ' ' && iter != '\t') {
		in_leading_space = false;
		}
		++iter;
		}

		// invariant: *iter == '\n' or iter == end

		// if this line has all white space, write '\n'
		if (in_leading_space) {
		p = _PyBytesWriter_Prepare(&writer, p, 1);
		if (p == NULL) {
		goto error;
		}
		*p++ = '\n';
		continue;
		}

		// copy [new_line_start + candidate_len, iter) to buffer, then append
		// '\n'
		new_line_len = iter - line_start - candidate_len;
		assert(new_line_len >= 0);
		p = _PyBytesWriter_Prepare(&writer, p, new_line_len + 1);
		if (p == NULL) {
		goto error;
		}
		memcpy(p, line_start + candidate_len, new_line_len);

		p += new_line_len;

		// this may always append '\n' at the end of `new_bytes`
		*p++ = '\n';
		int ret = _PyBytes_Dedent(start, nchars, &writer, &p);
		if (ret < 0) {
		return NULL;
		}

		new_bytes = _PyBytesWriter_Finish(&writer, p);
		if (new_bytes == NULL) {
		goto error;
		if (ret == 0) {
		Py_INCREF(bytes);
		_PyBytesWriter_Dealloc(&writer);
		return bytes;
		}
		Py_DECREF(bytes);
		return new_bytes;

		error:
		_PyBytesWriter_Dealloc(&writer);
		return NULL;
		return _PyBytesWriter_Finish(&writer, p);
		}

		static int
Expand All		@@ -396,10 +282,12 @@ pymain_run_command(wchar_t *command)
		goto error;
		}

		bytes = dedent_utf8_bytes(bytes);
		if (bytes == NULL) {
		PyObject *new_bytes = dedent_utf8_bytes(bytes);
		if (new_bytes == NULL) {
		Py_DECREF(bytes);
		goto error;
		}
		Py_SETREF(bytes, new_bytes);

		PyCompilerFlags cf = _PyCompilerFlags_INIT;
		cf.cf_flags \|= PyCF_IGNORE_COOKIE;
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -3535,6 +3535,8 @@ _PyBytesWriter_WriteBytes(_PyBytesWriter writer, void ptr,
		}


		/* Algorithms on bytes */

sunmy2019 marked this conversation as resolved. OutdatedShow resolvedHide resolved
		void
		_PyBytes_Repeat(char* dest, Py_ssize_t len_dest,
		const char* src, Py_ssize_t len_src)
Expand All		@@ -3558,3 +3560,148 @@ _PyBytes_Repeat(char* dest, Py_ssize_t len_dest,
		}
		}

sunmy2019 marked this conversation as resolved. Show resolvedHide resolved
		/** Dedent a UTF-8 encoded string.
methane marked this conversation as resolved. OutdatedShow resolvedHide resolved
		* behavior is expected to match `textwrap.dedent`
		*
		* return value:
		* 0, no need to dedent, writer untouched
		* 1, success
		* -1, failure
		*
		* str is the beginning of the string to dedent.
		* expecting (str != NULL)
		*
		* len is the length of the string to dedent.
		* expecting (len >= 0)
		*
sunmy2019 marked this conversation as resolved. OutdatedShow resolvedHide resolved
		* writer is a _PyBytesWriter object to write the dedented string.
		* expecting (writer != NULL)
		*
		* p points to a char* indicating the current position in the _PyBytesWriter.
		* It is updated to the new position after writing the dedented string on exit.
		* expecting (p != NULL && *p != NULL)
		*/
		int
		_PyBytes_Dedent(const char str, Py_ssize_t len, _PyBytesWriter writer,
		char **p)
sunmy2019 marked this conversation as resolved. OutdatedShow resolvedHide resolved
		{
sunmy2019 marked this conversation as resolved. OutdatedShow resolvedHide resolved
		assert(str);
		assert(p != NULL && *p != NULL);
		assert(writer);

		if (len <= 0)
		return 0;

		const char *end = str + len;
		assert(str < end); // prevent overflow when len is too large

		const char *candidate_start = NULL;
		Py_ssize_t candidate_len = 0;

		for (const char *iter = str; iter < end; ++iter) {
		const char *line_start = iter;
		const char *leading_whitespace_end = NULL;

		// scan the whole line
		while (iter < end && *iter != '\n') {
		if (!leading_whitespace_end && iter != ' ' && iter != '\t') {
		if (iter == line_start) {
		// some line has no indent, fast exit!
		return 0;
		}
		leading_whitespace_end = iter;
		}
		++iter;
		}

		// if this line has all white space, skip it
		if (!leading_whitespace_end) {
		continue;
		}

		if (!candidate_start) {
		candidate_start = line_start;
		candidate_len = leading_whitespace_end - line_start;
		assert(candidate_len > 0);
		} else {
		/* We then compare with the current longest leading whitespace.

		[line_start, leading_whitespace_end) is the leading whitespace of
		this line,

		[candidate_start, candidate_start + candidate_len)
		is the leading whitespace of the current longest leading
		whitespace. */
		Py_ssize_t new_candidate_len = 0;

		for (const char *candidate_iter = candidate_start,
		*line_iter = line_start;
		candidate_iter < candidate_start + candidate_len &&
		line_iter < leading_whitespace_end;
		++candidate_iter, ++line_iter) {
		if (candidate_iter != line_iter) {
		break;
		}
		++new_candidate_len;
		}

		candidate_len = new_candidate_len;
		if (candidate_len == 0) {
		return 0;
		}
		}
		}

		assert(candidate_len >= 0);
		if (candidate_len == 0) {
		return 0;
		}

		// trigger a dedent

		// prepare the writer
		char p_ = _PyBytesWriter_Prepare(writer, p, len);
		if (p_ == NULL) {
		*p = NULL;
		return -1;
		}

		for (const char *iter = str; iter < end; ++iter) {
		const char *line_start = iter;
		bool in_leading_space = true;

		// iterate over a line to find the end of a line
		while (iter < end && *iter != '\n') {
		if (in_leading_space && iter != ' ' && iter != '\t') {
		in_leading_space = false;
		}
		++iter;
		}

		// invariant: *iter == '\n' or iter == end
		bool append_newline = iter < end;

		// if this line has all white space, write '\n'
		if (in_leading_space && append_newline) {
		*p_++ = '\n';
		continue;
		}

		/* copy [new_line_start + candidate_len, iter) to buffer, then
		conditionally append '\n' */

		Py_ssize_t new_line_len = iter - line_start - candidate_len;
		assert(new_line_len >= 0);

		memcpy(p_, line_start + candidate_len, new_line_len);

		p_ += new_line_len;

		if (append_newline) {
		*p_++ = '\n';
		}
		}

		*p = p_;
		return 1;
		}