Oct 5, 2024 · Oct 5, 2024 · Oct 5, 2024 · Oct 6, 2024 · Oct 6, 2024 · Oct 6, 2024
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
 Lib/test/test_pyrepl/*        @pablogsal @lysnikolaou @ambv
 Tools/c-analyzer/             @ericsnowcurrently

 # cpuinfo
 Python/cpuinfo.c              @picnixz
 Python/pycore_cpuinfo*.h      @picnixz

 # dbm
 **/*dbm*                      @corona10 @erlend-aasland @serhiy-storchaka

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
 free-threading:${{ matrix.free-threading }}
 os:${{ matrix.os }}

 build-ubuntu-ssltests:
 build-ubuntu-ssltests-openssl:
 name:'Ubuntu SSL tests with OpenSSL'
 runs-on:${{ matrix.os }}
 timeout-minutes:60
    -name:SSL tests
 run:./python Lib/test/ssltests.py

 build-ubuntu-ssltests-awslc:
 name:'Ubuntu SSL tests with AWS-LC'
 runs-on:${{ matrix.os }}
 timeout-minutes:60
 needs:build-context
 if:needs.build-context.outputs.run-tests == 'true'
 strategy:
 fail-fast:false
 matrix:
 os:[ubuntu-24.04]
 awslc_ver:[1.55.0]
 env:
 AWSLC_VER:${{ matrix.awslc_ver}}
 MULTISSL_DIR:${{ github.workspace }}/multissl
 OPENSSL_DIR:${{ github.workspace }}/multissl/aws-lc/${{ matrix.awslc_ver }}
 LD_LIBRARY_PATH:${{ github.workspace }}/multissl/aws-lc/${{ matrix.awslc_ver }}/lib
 steps:
    -uses:actions/checkout@v4
 with:
 persist-credentials:false
    -name:Runner image version
 run:echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV"
    -name:Restore config.cache
 uses:actions/cache@v4
 with:
 path:config.cache
 key:${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }}
    -name:Register gcc problem matcher
 run:echo "::add-matcher::.github/problem-matchers/gcc.json"
    -name:Install dependencies
 run:sudo ./.github/workflows/posix-deps-apt.sh
    -name:Configure SSL lib env vars
 run:|
        echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV"
        echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/aws-lc/${AWSLC_VER}" >> "$GITHUB_ENV"
        echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/aws-lc/${AWSLC_VER}/lib" >> "$GITHUB_ENV"
    -name:'Restore AWS-LC build'
 id:cache-aws-lc
 uses:actions/cache@v4
 with:
 path:./multissl/aws-lc/${{ matrix.awslc_ver }}
 key:${{ matrix.os }}-multissl-aws-lc-${{ matrix.awslc_ver }}
    -name:Install AWS-LC
 if:steps.cache-aws-lc.outputs.cache-hit != 'true'
 run:|
        python3 Tools/ssl/multissltests.py \
          --steps=library \
          --base-directory "$MULTISSL_DIR" \
          --awslc ${{ matrix.awslc_ver }} \
          --system Linux
    -name:Add ccache to PATH
 run:|
        echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV"
    -name:Configure ccache action
 uses:hendrikmuhs/ccache-action@v1.2
 with:
 save:false
    -name:Configure CPython
 run:|
        ./configure CFLAGS="-fdiagnostics-format=json" \
          --config-cache \
          --enable-slower-safety \
          --with-pydebug \
          --with-openssl="$OPENSSL_DIR" \
          --with-builtin-hashlib-hashes=blake2 \
          --with-ssl-default-suites=openssl
    -name:Build CPython
 run:make -j
    -name:Display build info
 run:make pythoninfo
    -name:Verify python is linked to AWS-LC
 run:./python -c 'import ssl; print(ssl.OPENSSL_VERSION)' | grep AWS-LC
    -name:SSL tests
 run:./python Lib/test/ssltests.py

 build-wasi:
 name:'WASI'
 needs:build-context
    -build-windows-msi
    -build-macos
    -build-ubuntu
    -build-ubuntu-ssltests
    -build-ubuntu-ssltests-awslc
    -build-ubuntu-ssltests-openssl
    -build-wasi
    -test-hypothesis
    -build-asan
 with:
 allowed-failures:>-
          build-windows-msi,
          build-ubuntu-ssltests,
          build-ubuntu-ssltests-awslc,
          build-ubuntu-ssltests-openssl,
          test-hypothesis,
          cifuzz,
 allowed-skips:>-
            check-generated-files,
            build-macos,
            build-ubuntu,
            build-ubuntu-ssltests,
            build-ubuntu-ssltests-awslc,
            build-ubuntu-ssltests-openssl,
            build-wasi,
            test-hypothesis,
            build-asan,
diff --git a/Include/internal/pycore_cpuinfo.h b/Include/internal/pycore_cpuinfo.h
 /*
 * @author Bénédikt Tran
 *
 * Interface for detecting the different CPUID flags in an opaque manner.
 * See https://en.wikipedia.org/wiki/CPUID for details on the bit values.
 *
 * If a module requires to support SIMD instructions, it should determine
 * the compiler flags and the instruction sets required for the intrinsics
 * to work.
 *
 * For the headers and expected CPUID bits needed by Intel intrinsics, see
 * https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html.
 */

 #ifndef Py_INTERNAL_CPUINFO_H
 #define Py_INTERNAL_CPUINFO_H

 #ifdef __cplusplus
 extern "C" {
 #endif

 #ifndef Py_BUILD_CORE
 #  error "this header requires Py_BUILD_CORE define"
 #endif

 #include "Python.h"
 #include "pycore_cpuinfo_cpuid_features.h"

 typedef struct _Py_cpuid_features_s {
    uint32_t maxleaf;
    /*
     * Macro to declare a member flag of '_Py_cpuid_features' as a uint8_t.
     * Whenever this macro is used, do not forget to update the number of
     * fields and the bitsize of the 'ready' member (see structure end).
     */
 #define _Py_CPUID_DECL_FLAG(MEMBER_NAME)    uint8_t MEMBER_NAME:1
    // --- Streaming SIMD Extensions ------------------------------------------
    _Py_CPUID_DECL_FLAG(sse);
    _Py_CPUID_DECL_FLAG(sse2);
    _Py_CPUID_DECL_FLAG(sse3);
    _Py_CPUID_DECL_FLAG(ssse3); // Supplemental SSE3 instructions
    _Py_CPUID_DECL_FLAG(sse41); // SSE4.1
    _Py_CPUID_DECL_FLAG(sse42); // SSE4.2

    // --- Advanced Vector Extensions -----------------------------------------
    _Py_CPUID_DECL_FLAG(avx);
    _Py_CPUID_DECL_FLAG(avx_ifma);
    _Py_CPUID_DECL_FLAG(avx_ne_convert);

    _Py_CPUID_DECL_FLAG(avx_vnni);
    _Py_CPUID_DECL_FLAG(avx_vnni_int8);
    _Py_CPUID_DECL_FLAG(avx_vnni_int16);

    // --- Advanced Vector Extensions 2 ---------------------------------------
    _Py_CPUID_DECL_FLAG(avx2);

    // --- Advanced Vector Extensions (512-bit) -------------------------------
    /*
     * AVX-512 instruction set are grouped by the processor generation
     * that implements them (see https://en.wikipedia.org/wiki/AVX-512).
     *
     * We do not include GFNI, VPCLMULQDQ and VAES instructions since
     * they are not exactly AVX-512 per se, nor do we include BF16 or
     * FP16 since they operate on bfloat16 and binary16 (half-float).
     *
     * See https://en.wikipedia.org/wiki/AVX-512#Instruction_set for
     * the suffix meanings (for instance 'f' stands for 'Foundation').
     */
    _Py_CPUID_DECL_FLAG(avx512_f);
    _Py_CPUID_DECL_FLAG(avx512_cd);

    _Py_CPUID_DECL_FLAG(avx512_er);
    _Py_CPUID_DECL_FLAG(avx512_pf);

    _Py_CPUID_DECL_FLAG(avx512_4fmaps);
    _Py_CPUID_DECL_FLAG(avx512_4vnniw);

    _Py_CPUID_DECL_FLAG(avx512_vpopcntdq);

    _Py_CPUID_DECL_FLAG(avx512_vl);
    _Py_CPUID_DECL_FLAG(avx512_dq);
    _Py_CPUID_DECL_FLAG(avx512_bw);

    _Py_CPUID_DECL_FLAG(avx512_ifma);
    _Py_CPUID_DECL_FLAG(avx512_vbmi);

    _Py_CPUID_DECL_FLAG(avx512_vnni);

    _Py_CPUID_DECL_FLAG(avx512_vbmi2);
    _Py_CPUID_DECL_FLAG(avx512_bitalg);

    _Py_CPUID_DECL_FLAG(avx512_vp2intersect);

    // --- Instructions -------------------------------------------------------
    _Py_CPUID_DECL_FLAG(cmov);
    _Py_CPUID_DECL_FLAG(fma);
    _Py_CPUID_DECL_FLAG(popcnt);
    _Py_CPUID_DECL_FLAG(pclmulqdq);

    _Py_CPUID_DECL_FLAG(xsave);   // XSAVE/XRSTOR/XSETBV/XGETBV
    _Py_CPUID_DECL_FLAG(osxsave); // XSAVE is enabled by the OS

 #undef _Py_CPUID_DECL_FLAG
    // Whenever a field is added or removed above, update the
    // number of fields (35) and adjust the bitsize of 'ready'
    // so that the size of this structure is a multiple of 8.
    uint8_t ready: 5;  // set if the structure is ready for usage
 } _Py_cpuid_features;

 /*
 * Explicitly set all members to zero to guarantee that
 * we never have a non-initialized attribute at runtime
 * which could lead to an illegal instruction error.
 *
 * This readiness state of 'flags' is ignored and left untouched.
 *
 * Note: This function does not set any exception and thus never fails.
 */
 PyAPI_FUNC(void)
 _Py_cpuid_disable_features(_Py_cpuid_features *flags);

 /*
 * Check whether the structure is ready and flags are inter-compatible,
 * returning 1 on success and 0 otherwise.
 *
 * The caller should disable all CPUID detected features if the check
 * fails to avoid encountering runtime illegal instruction errors.
 *
 * Note: This function does not set any exception and thus never fails.
 */
 PyAPI_FUNC(int)
 _Py_cpuid_check_features(const _Py_cpuid_features *flags);

 /*
 * Return 1 if all expected flags are set in 'actual', 0 otherwise.
 *
 * If 'actual' or 'expect' are not ready yet, this also returns 0.
 *
 * Note: This function does not set any exception and thus never fails.
 */
 PyAPI_FUNC(int)
 _Py_cpuid_has_features(const _Py_cpuid_features *actual,
                       const _Py_cpuid_features *expect);

 /*
 * Return 1 if 'actual' and 'expect' are identical, 0 otherwise.
 *
 * If 'actual' or 'expect' are not ready yet, this also returns 0.
 *
 * Note: This function does not set any exception and thus never fails.
 */
 PyAPI_FUNC(int)
 _Py_cpuid_match_features(const _Py_cpuid_features *actual,
                         const _Py_cpuid_features *expect);

 /*
 * Detect the available host features, storing the result in 'flags'.
 *
 * Note: This function does not set any exception and thus never fails.
 */
 PyAPI_FUNC(void)
 _Py_cpuid_detect_features(_Py_cpuid_features *flags);

 #ifdef __cplusplus
 }
 #endif

 #endif /* !Py_INTERNAL_CPUINFO_H */
Original file line number	Diff line number	Diff line change
Expand Up		@@ -56,6 +56,10 @@ Lib/test/test_capi/test_misc.py @markshannon
		Lib/test/test_pyrepl/* @pablogsal @lysnikolaou @ambv
		Tools/c-analyzer/ @ericsnowcurrently

		# cpuinfo
		Python/cpuinfo.c @picnixz
		Python/pycore_cpuinfo*.h @picnixz

		# dbm
		*/dbm* @corona10 @erlend-aasland @serhiy-storchaka

Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -260,7 +260,7 @@ jobs:
		free-threading:${{ matrix.free-threading }}
		os:${{ matrix.os }}

		build-ubuntu-ssltests:
		build-ubuntu-ssltests-openssl:
		name:'Ubuntu SSL tests with OpenSSL'
		runs-on:${{ matrix.os }}
		timeout-minutes:60
Expand DownExpand Up		@@ -322,6 +322,81 @@ jobs:
		-name:SSL tests
		run:./python Lib/test/ssltests.py

		build-ubuntu-ssltests-awslc:
		name:'Ubuntu SSL tests with AWS-LC'
		runs-on:${{ matrix.os }}
		timeout-minutes:60
		needs:build-context
		if:needs.build-context.outputs.run-tests == 'true'
		strategy:
		fail-fast:false
		matrix:
		os:[ubuntu-24.04]
		awslc_ver:[1.55.0]
		env:
		AWSLC_VER:${{ matrix.awslc_ver}}
		MULTISSL_DIR:${{ github.workspace }}/multissl
		OPENSSL_DIR:${{ github.workspace }}/multissl/aws-lc/${{ matrix.awslc_ver }}
		LD_LIBRARY_PATH:${{ github.workspace }}/multissl/aws-lc/${{ matrix.awslc_ver }}/lib
		steps:
		-uses:actions/checkout@v4
		with:
		persist-credentials:false
		-name:Runner image version
		run:echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV"
		-name:Restore config.cache
		uses:actions/cache@v4
		with:
		path:config.cache
		key:${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }}
		-name:Register gcc problem matcher
		run:echo "::add-matcher::.github/problem-matchers/gcc.json"
		-name:Install dependencies
		run:sudo ./.github/workflows/posix-deps-apt.sh
		-name:Configure SSL lib env vars
		run:\|
		echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV"
		echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/aws-lc/${AWSLC_VER}" >> "$GITHUB_ENV"
		echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/aws-lc/${AWSLC_VER}/lib" >> "$GITHUB_ENV"
		-name:'Restore AWS-LC build'
		id:cache-aws-lc
		uses:actions/cache@v4
		with:
		path:./multissl/aws-lc/${{ matrix.awslc_ver }}
		key:${{ matrix.os }}-multissl-aws-lc-${{ matrix.awslc_ver }}
		-name:Install AWS-LC
		if:steps.cache-aws-lc.outputs.cache-hit != 'true'
		run:\|
		python3 Tools/ssl/multissltests.py \
		--steps=library \
		--base-directory "$MULTISSL_DIR" \
		--awslc ${{ matrix.awslc_ver }} \
		--system Linux
		-name:Add ccache to PATH
		run:\|
		echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV"
		-name:Configure ccache action
		uses:hendrikmuhs/ccache-action@v1.2
		with:
		save:false
		-name:Configure CPython
		run:\|
		./configure CFLAGS="-fdiagnostics-format=json" \
		--config-cache \
		--enable-slower-safety \
		--with-pydebug \
		--with-openssl="$OPENSSL_DIR" \
		--with-builtin-hashlib-hashes=blake2 \
		--with-ssl-default-suites=openssl
		-name:Build CPython
		run:make -j
		-name:Display build info
		run:make pythoninfo
		-name:Verify python is linked to AWS-LC
		run:./python -c 'import ssl; print(ssl.OPENSSL_VERSION)' \| grep AWS-LC
		-name:SSL tests
		run:./python Lib/test/ssltests.py

		build-wasi:
		name:'WASI'
		needs:build-context
Expand DownExpand Up		@@ -628,7 +703,8 @@ jobs:
		-build-windows-msi
		-build-macos
		-build-ubuntu
		-build-ubuntu-ssltests
		-build-ubuntu-ssltests-awslc
		-build-ubuntu-ssltests-openssl
		-build-wasi
		-test-hypothesis
		-build-asan
Expand All		@@ -643,7 +719,8 @@ jobs:
		with:
		allowed-failures:>-
		build-windows-msi,
		build-ubuntu-ssltests,
		build-ubuntu-ssltests-awslc,
		build-ubuntu-ssltests-openssl,
		test-hypothesis,
		cifuzz,
		allowed-skips:>-
Expand All		@@ -661,7 +738,8 @@ jobs:
		check-generated-files,
		build-macos,
		build-ubuntu,
		build-ubuntu-ssltests,
		build-ubuntu-ssltests-awslc,
		build-ubuntu-ssltests-openssl,
		build-wasi,
		test-hypothesis,
		build-asan,
Expand Down
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,168 @@
		/*
		* @author Bénédikt Tran
		*
		* Interface for detecting the different CPUID flags in an opaque manner.
		* See https://en.wikipedia.org/wiki/CPUID for details on the bit values.
		*
		* If a module requires to support SIMD instructions, it should determine
		* the compiler flags and the instruction sets required for the intrinsics
		* to work.
		*
		* For the headers and expected CPUID bits needed by Intel intrinsics, see
		* https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html.
		*/

		#ifndef Py_INTERNAL_CPUINFO_H
		#define Py_INTERNAL_CPUINFO_H

		#ifdef __cplusplus
		extern "C" {
		#endif

		#ifndef Py_BUILD_CORE
		# error "this header requires Py_BUILD_CORE define"
		#endif

		#include "Python.h"
		#include "pycore_cpuinfo_cpuid_features.h"

		typedef struct _Py_cpuid_features_s {
		uint32_t maxleaf;
		/*
		* Macro to declare a member flag of '_Py_cpuid_features' as a uint8_t.
		* Whenever this macro is used, do not forget to update the number of
		* fields and the bitsize of the 'ready' member (see structure end).
		*/
		#define _Py_CPUID_DECL_FLAG(MEMBER_NAME) uint8_t MEMBER_NAME:1
		// --- Streaming SIMD Extensions ------------------------------------------
		_Py_CPUID_DECL_FLAG(sse);
		_Py_CPUID_DECL_FLAG(sse2);
		_Py_CPUID_DECL_FLAG(sse3);
		_Py_CPUID_DECL_FLAG(ssse3); // Supplemental SSE3 instructions
		_Py_CPUID_DECL_FLAG(sse41); // SSE4.1
		_Py_CPUID_DECL_FLAG(sse42); // SSE4.2

		// --- Advanced Vector Extensions -----------------------------------------
		_Py_CPUID_DECL_FLAG(avx);
		_Py_CPUID_DECL_FLAG(avx_ifma);
		_Py_CPUID_DECL_FLAG(avx_ne_convert);

		_Py_CPUID_DECL_FLAG(avx_vnni);
		_Py_CPUID_DECL_FLAG(avx_vnni_int8);
		_Py_CPUID_DECL_FLAG(avx_vnni_int16);

		// --- Advanced Vector Extensions 2 ---------------------------------------
		_Py_CPUID_DECL_FLAG(avx2);

		// --- Advanced Vector Extensions (512-bit) -------------------------------
		/*
		* AVX-512 instruction set are grouped by the processor generation
		* that implements them (see https://en.wikipedia.org/wiki/AVX-512).
		*
		* We do not include GFNI, VPCLMULQDQ and VAES instructions since
		* they are not exactly AVX-512 per se, nor do we include BF16 or
		* FP16 since they operate on bfloat16 and binary16 (half-float).
		*
		* See https://en.wikipedia.org/wiki/AVX-512#Instruction_set for
		* the suffix meanings (for instance 'f' stands for 'Foundation').
		*/
		_Py_CPUID_DECL_FLAG(avx512_f);
		_Py_CPUID_DECL_FLAG(avx512_cd);

		_Py_CPUID_DECL_FLAG(avx512_er);
		_Py_CPUID_DECL_FLAG(avx512_pf);

		_Py_CPUID_DECL_FLAG(avx512_4fmaps);
		_Py_CPUID_DECL_FLAG(avx512_4vnniw);

		_Py_CPUID_DECL_FLAG(avx512_vpopcntdq);

		_Py_CPUID_DECL_FLAG(avx512_vl);
		_Py_CPUID_DECL_FLAG(avx512_dq);
		_Py_CPUID_DECL_FLAG(avx512_bw);

		_Py_CPUID_DECL_FLAG(avx512_ifma);
		_Py_CPUID_DECL_FLAG(avx512_vbmi);

		_Py_CPUID_DECL_FLAG(avx512_vnni);

		_Py_CPUID_DECL_FLAG(avx512_vbmi2);
		_Py_CPUID_DECL_FLAG(avx512_bitalg);

		_Py_CPUID_DECL_FLAG(avx512_vp2intersect);

		// --- Instructions -------------------------------------------------------
		_Py_CPUID_DECL_FLAG(cmov);
		_Py_CPUID_DECL_FLAG(fma);
		_Py_CPUID_DECL_FLAG(popcnt);
		_Py_CPUID_DECL_FLAG(pclmulqdq);

		_Py_CPUID_DECL_FLAG(xsave); // XSAVE/XRSTOR/XSETBV/XGETBV
		_Py_CPUID_DECL_FLAG(osxsave); // XSAVE is enabled by the OS

		#undef _Py_CPUID_DECL_FLAG
		// Whenever a field is added or removed above, update the
		// number of fields (35) and adjust the bitsize of 'ready'
		// so that the size of this structure is a multiple of 8.
		uint8_t ready: 5; // set if the structure is ready for usage
		} _Py_cpuid_features;

		/*
		* Explicitly set all members to zero to guarantee that
		* we never have a non-initialized attribute at runtime
		* which could lead to an illegal instruction error.
		*
		* This readiness state of 'flags' is ignored and left untouched.
		*
		* Note: This function does not set any exception and thus never fails.
		*/
		PyAPI_FUNC(void)
		_Py_cpuid_disable_features(_Py_cpuid_features *flags);

		/*
		* Check whether the structure is ready and flags are inter-compatible,
		* returning 1 on success and 0 otherwise.
		*
		* The caller should disable all CPUID detected features if the check
		* fails to avoid encountering runtime illegal instruction errors.
		*
		* Note: This function does not set any exception and thus never fails.
		*/
		PyAPI_FUNC(int)
		_Py_cpuid_check_features(const _Py_cpuid_features *flags);

		/*
		* Return 1 if all expected flags are set in 'actual', 0 otherwise.
		*
		* If 'actual' or 'expect' are not ready yet, this also returns 0.
		*
		* Note: This function does not set any exception and thus never fails.
		*/
		PyAPI_FUNC(int)
		_Py_cpuid_has_features(const _Py_cpuid_features *actual,
		const _Py_cpuid_features *expect);

		/*
		* Return 1 if 'actual' and 'expect' are identical, 0 otherwise.
		*
		* If 'actual' or 'expect' are not ready yet, this also returns 0.
		*
		* Note: This function does not set any exception and thus never fails.
		*/
		PyAPI_FUNC(int)
		_Py_cpuid_match_features(const _Py_cpuid_features *actual,
		const _Py_cpuid_features *expect);

		/*
		* Detect the available host features, storing the result in 'flags'.
		*
		* Note: This function does not set any exception and thus never fails.
		*/
		PyAPI_FUNC(void)
		_Py_cpuid_detect_features(_Py_cpuid_features *flags);

		#ifdef __cplusplus
		}
		#endif

		#endif /* !Py_INTERNAL_CPUINFO_H */