Apr 18, 2025 · Apr 29, 2023 · Apr 29, 2023 · Apr 29, 2023 · Apr 30, 2023 · Apr 30, 2023
diff --git a/Include/internal/pycore_bytesobject.h b/Include/internal/pycore_bytesobject.h
 _PyBytes_Repeat(char* dest, Py_ssize_t len_dest,
    const char* src, Py_ssize_t len_src);


 /** Dedent a UTF-8 encoded string.
 * behavior is expected to match `textwrap.dedent`
 *
 * return value:
 * 0, no need to dedent, `dest` buffer and `*dest_len` untouched
 * 1, success
 *
 * `src` is the string to dedent.
 * expecting `(src != NULL)`
 *
 * `src_len` is the length of `src`.
 *
 * `dest` is a buffer for the result.
 * expecting `(dest != NULL)`
 *
 * `*dest_len` stores the length of `dest` on entry, and is updated to the
 * length of the dedent result upon success. Output buffer should be large
 * enough to hold the result.
 * expecting `(dest_len != NULL && *dest_len >= src_len)`
 */
 PyAPI_FUNC(int)
 _PyBytes_Dedent(const char *src, Py_ssize_t src_len, char* dest,
                Py_ssize_t* dest_len);

 /* --- _PyBytesWriter ----------------------------------------------------- */

 /* The _PyBytesWriter structure is big: it contains an embedded "stack buffer".
diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h
    PyObject *thousands_sep,
    Py_UCS4 *maxchar);

 /* Dedent a string.
   Behaviour is expected to be an exact match of `textwrap.dedent`.
   Return a new reference on success, NULL with exception set on error.
   */
 PyAPI_FUNC(PyObject*) _PyUnicode_Dedent(PyObject *unicode);

 /* --- Misc functions ----------------------------------------------------- */

 extern PyObject* _PyUnicode_FormatLong(PyObject *, int, int, int);
diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-29-23-15-38.gh-issue-103997.BS3uVt.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-29-23-15-38.gh-issue-103997.BS3uVt.rst
 String arguments passed to"-c" are now automatically dedented as if by
 String arguments passed to "-c" are now automatically dedented as if by
 :func:`textwrap.dedent`. This allows "python -c" invocations to be indented
 in shell scripts without causing indentation errors.
 in shell scripts without causing indentation errors. (Patch by Jon Crall and
 Steven Sun)

 Add a private API :c:func:`_PyBytes_Dedent`.

 (Patch by Jon Crall and Steven Sun)
 Add an internal API :c:func:`_PyUnicode_Dedent`. (Patch by Steven Sun)
diff --git a/Modules/main.c b/Modules/main.c
    }
 }

 /* Strip common leading whitespace, just as textwrap.dedent.
   It returns a new reference. */
 static PyObject *
 dedent_utf8_bytes(PyObject *bytes)
 {
    assert(bytes != NULL && PyBytes_CheckExact(bytes));

    Py_ssize_t nchars;
    char *start;
    if (PyBytes_AsStringAndSize(bytes, &start, &nchars) != 0) {
        return NULL;
    }

    char* p = PyMem_Malloc(nchars);
    if (p == NULL) {
        PyErr_NoMemory();
        return NULL;
    }

    int ret = _PyBytes_Dedent(start, nchars, p, &nchars);

    if (ret == 0) {
        Py_INCREF(bytes);
        PyMem_Free(p);
        return bytes;
    }

    PyObject* new_bytes = PyBytes_FromStringAndSize(p, nchars);
    PyMem_Free(p);
    return new_bytes;
 }

 static int
 pymain_run_command(wchar_t *command)
        return pymain_exit_err_print();
    }

    bytes = PyUnicode_AsUTF8String(unicode);
    Py_DECREF(unicode);
    if (bytes == NULL) {
    Py_SETREF(unicode, _PyUnicode_Dedent(unicode));
    if (unicode == NULL) {
        goto error;
    }

 PyObject *new_bytes = dedent_utf8_bytes(bytes);
 if (new_bytes == NULL) {
    Py_DECREF(bytes);
 bytes = PyUnicode_AsUTF8String(unicode);
 Py_DECREF(unicode);
 if(bytes == NULL) {
        goto error;
    }
    Py_SETREF(bytes, new_bytes);

    PyCompilerFlags cf = _PyCompilerFlags_INIT;
    cf.cf_flags |= PyCF_IGNORE_COOKIE;
diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c
        }
    }
 }

 /** Dedent a UTF-8 encoded string.
 * behavior is expected to match `textwrap.dedent`
 *
 * return value:
 * 0, no need to dedent, `dest` buffer and `*dest_len` untouched
 * 1, success
 *
 * `src` is the string to dedent.
 * expecting `(src != NULL)`
 *
 * `src_len` is the length of `src`.
 *
 * `dest` is a buffer for the result.
 * expecting `(dest != NULL)`
 *
 * `*dest_len` stores the length of `dest` on entry, and is updated to the
 * length of the dedent result upon success. Output buffer should be large
 * enough to hold the result.
 * expecting `(dest_len != NULL && *dest_len >= src_len)`
 */
 int
 _PyBytes_Dedent(const char *src, Py_ssize_t src_len, char *dest,
                Py_ssize_t *dest_len) {
    assert(src && dest && dest_len);
    assert(*dest_len >= src_len);

    if (src_len <= 0)
        return 0;

    const char *end = src + src_len;
    assert(src < end); // prevent overflow when src_len is too large

    const char *candidate_start = NULL;
    Py_ssize_t candidate_len = 0;

    for (const char *iter = src; iter < end; ++iter) {
        const char *line_start = iter;
        const char *leading_whitespace_end = NULL;

        // scan the whole line
        while (iter < end && *iter != '\n') {
            if (!leading_whitespace_end && *iter != ' ' && *iter != '\t') {
                if (iter == line_start) {
                    // some line has no indent, fast exit!
                    return 0;
                }
                leading_whitespace_end = iter;
            }
            ++iter;
        }

        // if this line has all white space, skip it
        if (!leading_whitespace_end) {
            continue;
        }

        if (!candidate_start) {
            candidate_start = line_start;
            candidate_len = leading_whitespace_end - line_start;
            assert(candidate_len > 0);
        } else {
            /* We then compare with the current longest leading whitespace.

               [line_start, leading_whitespace_end) is the leading whitespace of
               this line,

               [candidate_start, candidate_start + candidate_len)
               is the leading whitespace of the current longest leading
               whitespace. */
            Py_ssize_t new_candidate_len = 0;

            for (const char *candidate_iter = candidate_start,
                            *line_iter = line_start;
                 candidate_iter < candidate_start + candidate_len &&
                 line_iter < leading_whitespace_end;
                 ++candidate_iter, ++line_iter) {
                if (*candidate_iter != *line_iter) {
                    break;
                }
                ++new_candidate_len;
            }

            candidate_len = new_candidate_len;
            if (candidate_len == 0) {
                return 0;
            }
        }
    }

    assert(candidate_len >= 0);
    if (candidate_len == 0) {
        return 0;
    }

    // trigger a dedent
    char *dest_start = dest;

    for (const char *iter = src; iter < end; ++iter) {
        const char *line_start = iter;
        bool in_leading_space = true;

        // iterate over a line to find the end of a line
        while (iter < end && *iter != '\n') {
            if (in_leading_space && *iter != ' ' && *iter != '\t') {
                in_leading_space = false;
            }
            ++iter;
        }

        // invariant: *iter == '\n' or iter == end
        bool append_newline = iter < end;

        // if this line has all white space, write '\n'
        if (in_leading_space && append_newline) {
            *dest++ = '\n';
            continue;
        }

        /* copy [new_line_start + candidate_len, iter) to buffer, then
            conditionally append '\n' */

        Py_ssize_t new_line_len = iter - line_start - candidate_len;
        assert(new_line_len >= 0);

        memcpy(dest, line_start + candidate_len, new_line_len);

        dest += new_line_len;

        if (append_newline) {
            *dest++ = '\n';
        }
    }
    *dest_len = dest - dest_start;
    return 1;
 }
Original file line number	Diff line number	Diff line change
Expand Up		@@ -41,31 +41,6 @@ PyAPI_FUNC(void)
		_PyBytes_Repeat(char* dest, Py_ssize_t len_dest,
		const char* src, Py_ssize_t len_src);


		/** Dedent a UTF-8 encoded string.
		* behavior is expected to match `textwrap.dedent`
		*
		* return value:
		* 0, no need to dedent, `dest` buffer and `*dest_len` untouched
		* 1, success
		*
		* `src` is the string to dedent.
		* expecting `(src != NULL)`
		*
		* `src_len` is the length of `src`.
		*
		* `dest` is a buffer for the result.
		* expecting `(dest != NULL)`
		*
		* `*dest_len` stores the length of `dest` on entry, and is updated to the
		* length of the dedent result upon success. Output buffer should be large
		* enough to hold the result.
		* expecting `(dest_len != NULL && *dest_len >= src_len)`
		*/
		PyAPI_FUNC(int)
		_PyBytes_Dedent(const char src, Py_ssize_t src_len, char dest,
		Py_ssize_t* dest_len);

		/* --- _PyBytesWriter ----------------------------------------------------- */

		/* The _PyBytesWriter structure is big: it contains an embedded "stack buffer".
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -339,6 +339,12 @@ extern Py_ssize_t _PyUnicode_InsertThousandsGrouping(
		PyObject *thousands_sep,
		Py_UCS4 *maxchar);

		/* Dedent a string.
		Behaviour is expected to be an exact match of `textwrap.dedent`.
		Return a new reference on success, NULL with exception set on error.
		*/
		PyAPI_FUNC(PyObject) _PyUnicode_Dedent(PyObject unicode);
sunmy2019 marked this conversation as resolved. OutdatedShow resolvedHide resolved

		/* --- Misc functions ----------------------------------------------------- */

		extern PyObject* _PyUnicode_FormatLong(PyObject *, int, int, int);
Expand Down
Original file line number	Diff line number	Diff line change
		@@ -1,7 +1,6 @@
		String arguments passed to"-c" are now automatically dedented as if by
		String arguments passed to "-c" are now automatically dedented as if by
		:func:`textwrap.dedent`. This allows "python -c" invocations to be indented
		in shell scripts without causing indentation errors.
		in shell scripts without causing indentation errors. (Patch by Jon Crall and
		Steven Sun)

		Add a private API :c:func:`_PyBytes_Dedent`.

		(Patch by Jon Crall and Steven Sun)
		Add an internal API :c:func:`_PyUnicode_Dedent`. (Patch by Steven Sun)
Original file line number	Diff line number	Diff line change
Expand Up		@@ -229,37 +229,6 @@ pymain_import_readline(const PyConfig *config)
		}
		}

		/* Strip common leading whitespace, just as textwrap.dedent.
		It returns a new reference. */
		static PyObject *
		dedent_utf8_bytes(PyObject *bytes)
		{
		assert(bytes != NULL && PyBytes_CheckExact(bytes));

		Py_ssize_t nchars;
		char *start;
		if (PyBytes_AsStringAndSize(bytes, &start, &nchars) != 0) {
		return NULL;
		}

		char* p = PyMem_Malloc(nchars);
		if (p == NULL) {
		PyErr_NoMemory();
		return NULL;
		}

		int ret = _PyBytes_Dedent(start, nchars, p, &nchars);

		if (ret == 0) {
		Py_INCREF(bytes);
		PyMem_Free(p);
		return bytes;
		}

		PyObject* new_bytes = PyBytes_FromStringAndSize(p, nchars);
		PyMem_Free(p);
		return new_bytes;
		}

		static int
		pymain_run_command(wchar_t *command)
Expand All		@@ -276,18 +245,16 @@ pymain_run_command(wchar_t *command)
		return pymain_exit_err_print();
		}

		bytes = PyUnicode_AsUTF8String(unicode);
		Py_DECREF(unicode);
		if (bytes == NULL) {
		Py_SETREF(unicode, _PyUnicode_Dedent(unicode));
		if (unicode == NULL) {
		goto error;
		}

		PyObject *new_bytes = dedent_utf8_bytes(bytes);
		if (new_bytes == NULL) {
		Py_DECREF(bytes);
		bytes = PyUnicode_AsUTF8String(unicode);
		Py_DECREF(unicode);
		if(bytes == NULL) {
		goto error;
		}
		Py_SETREF(bytes, new_bytes);

		PyCompilerFlags cf = _PyCompilerFlags_INIT;
		cf.cf_flags \|= PyCF_IGNORE_COOKIE;
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -3559,139 +3559,3 @@ _PyBytes_Repeat(char* dest, Py_ssize_t len_dest,
		}
		}
		}

sunmy2019 marked this conversation as resolved. Show resolvedHide resolved
		/** Dedent a UTF-8 encoded string.
		* behavior is expected to match `textwrap.dedent`
		*
		* return value:
		* 0, no need to dedent, `dest` buffer and `*dest_len` untouched
		* 1, success
		*
		* `src` is the string to dedent.
		* expecting `(src != NULL)`
		*
		* `src_len` is the length of `src`.
		*
		* `dest` is a buffer for the result.
		* expecting `(dest != NULL)`
		*
		* `*dest_len` stores the length of `dest` on entry, and is updated to the
		* length of the dedent result upon success. Output buffer should be large
		* enough to hold the result.
		* expecting `(dest_len != NULL && *dest_len >= src_len)`
		*/
		int
		_PyBytes_Dedent(const char src, Py_ssize_t src_len, char dest,
		Py_ssize_t *dest_len) {
		assert(src && dest && dest_len);
		assert(*dest_len >= src_len);

		if (src_len <= 0)
		return 0;

		const char *end = src + src_len;
		assert(src < end); // prevent overflow when src_len is too large

		const char *candidate_start = NULL;
		Py_ssize_t candidate_len = 0;

		for (const char *iter = src; iter < end; ++iter) {
		const char *line_start = iter;
		const char *leading_whitespace_end = NULL;

		// scan the whole line
		while (iter < end && *iter != '\n') {
		if (!leading_whitespace_end && iter != ' ' && iter != '\t') {
		if (iter == line_start) {
		// some line has no indent, fast exit!
		return 0;
		}
		leading_whitespace_end = iter;
		}
		++iter;
		}

		// if this line has all white space, skip it
		if (!leading_whitespace_end) {
		continue;
		}

		if (!candidate_start) {
		candidate_start = line_start;
		candidate_len = leading_whitespace_end - line_start;
		assert(candidate_len > 0);
		} else {
		/* We then compare with the current longest leading whitespace.

		[line_start, leading_whitespace_end) is the leading whitespace of
		this line,

		[candidate_start, candidate_start + candidate_len)
		is the leading whitespace of the current longest leading
		whitespace. */
		Py_ssize_t new_candidate_len = 0;

		for (const char *candidate_iter = candidate_start,
		*line_iter = line_start;
		candidate_iter < candidate_start + candidate_len &&
		line_iter < leading_whitespace_end;
		++candidate_iter, ++line_iter) {
		if (candidate_iter != line_iter) {
		break;
		}
		++new_candidate_len;
		}

		candidate_len = new_candidate_len;
		if (candidate_len == 0) {
		return 0;
		}
		}
		}

		assert(candidate_len >= 0);
		if (candidate_len == 0) {
		return 0;
		}

		// trigger a dedent
		char *dest_start = dest;

		for (const char *iter = src; iter < end; ++iter) {
		const char *line_start = iter;
		bool in_leading_space = true;

		// iterate over a line to find the end of a line
		while (iter < end && *iter != '\n') {
		if (in_leading_space && iter != ' ' && iter != '\t') {
		in_leading_space = false;
		}
		++iter;
		}

		// invariant: *iter == '\n' or iter == end
		bool append_newline = iter < end;

		// if this line has all white space, write '\n'
		if (in_leading_space && append_newline) {
		*dest++ = '\n';
		continue;
		}

		/* copy [new_line_start + candidate_len, iter) to buffer, then
		conditionally append '\n' */

		Py_ssize_t new_line_len = iter - line_start - candidate_len;
		assert(new_line_len >= 0);

		memcpy(dest, line_start + candidate_len, new_line_len);

		dest += new_line_len;

		if (append_newline) {
		*dest++ = '\n';
		}
		}
		*dest_len = dest - dest_start;
		return 1;
		}