From gniibe at fsij.org  Wed Jun  3 07:47:00 2026
From: gniibe at fsij.org (NIIBE Yutaka)
Date: Wed,  3 Jun 2026 14:47:00 +0900
Subject: [PATCH] sexp: Add length check DATALEN when parsing SEXP.
Message-ID: <30365d270bc6aa7dc8a03c8693c7fa086defcc2c.1780465526.git.gniibe@fsij.org>


* src/sexp.c (do_vsexp_sscan): Raise an error when the length
cannot be represented by the DATALEN type.

--

Reported-by: Ciwan ?ztopal
GnuPG-bug-id: 8267
Signed-off-by: NIIBE Yutaka <gniibe at fsij.org>
---
 src/sexp.c | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 56 insertions(+), 2 deletions(-)

-------------- next part --------------
A non-text attachment was scrubbed...
Name: 0001-sexp-Add-length-check-DATALEN-when-parsing-SEXP.patch
Type: text/x-patch
Size: 4110 bytes
Desc: not available
URL: <https://lists.gnupg.org/pipermail/gcrypt-devel/attachments/20260603/a9e79afd/attachment.bin>

From gniibe at fsij.org  Thu Jun  4 07:46:45 2026
From: gniibe at fsij.org (NIIBE Yutaka)
Date: Thu,  4 Jun 2026 14:46:45 +0900
Subject: [PATCH] cipher:kem: Validate input length.
Message-ID: <db5ae94eb01feb3cb60d42e6caa75106e3e781fc.1780551989.git.gniibe@fsij.org>


* cipher/kem.c (_gcry_kem_genkey): Add input length validation.
(_gcry_kem_encap, _gcry_kem_decap): Likewise.
* src/gcrypt.h.in: Add forgotten constants for X448.

--

Reported-by: Ciwan ?ztopal
GnuPG-bug-id: 8259
Signed-off-by: NIIBE Yutaka <gniibe at fsij.org>
---
 cipher/kem.c    | 124 ++++++++++++++++++++++++++++++++++++++++--------
 src/gcrypt.h.in |  13 +++++
 2 files changed, 118 insertions(+), 19 deletions(-)

-------------- next part --------------
A non-text attachment was scrubbed...
Name: 0001-cipher-kem-Validate-input-length.patch
Type: text/x-patch
Size: 9721 bytes
Desc: not available
URL: <https://lists.gnupg.org/pipermail/gcrypt-devel/attachments/20260604/12a55d2c/attachment-0001.bin>

From jussi.kivilinna at iki.fi  Sun Jun 28 14:37:35 2026
From: jussi.kivilinna at iki.fi (Jussi Kivilinna)
Date: Sun, 28 Jun 2026 15:37:35 +0300
Subject: [PATCH 1/7] Add straight-line speculation hardening for function ends
Message-ID: <20260628123742.1916-1-jussi.kivilinna@iki.fi>

* cipher/asm-common-amd64.h (SPEC_STOP): New.
(CFI_ENDPROC): Emit straight-line speculation barrier at function end.
(ret_spec_stop): Use 'SPEC_STOP'.
* cipher/asm-common-i386.h (SPEC_STOP, CFI_ENDPROC, ret_spec_stop):
Likewise.
* cipher/asm-common-aarch64.h (SPEC_STOP, CFI_ENDPROC, ret_spec_stop):
Likewise.
* cipher/cast5-amd64.S (__cast5_dec_blk4): Move 'CFI_ENDPROC' after
'ret_spec_stop'.
* mpi/i386/syntax.h (SPEC_STOP): New.
(ret_spec_stop): Use 'SPEC_STOP'.
--

The 'ret_spec_stop' macro places a speculation barrier after 'ret'
instructions only. Emit the barrier from 'CFI_ENDPROC' as well so that
the end of every function is covered, and move the barrier instruction
into a new 'SPEC_STOP' macro shared by both.

Signed-off-by: Jussi Kivilinna <jussi.kivilinna at iki.fi>
---
 cipher/asm-common-aarch64.h | 9 ++++++---
 cipher/asm-common-amd64.h   | 9 ++++++---
 cipher/asm-common-i386.h    | 9 ++++++---
 cipher/cast5-amd64.S        | 2 +-
 mpi/i386/syntax.h           | 5 ++++-
 5 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/cipher/asm-common-aarch64.h b/cipher/asm-common-aarch64.h
index dde7366c..ec9b9b92 100644
--- a/cipher/asm-common-aarch64.h
+++ b/cipher/asm-common-aarch64.h
@@ -61,10 +61,13 @@
 # define AARCH64_PAC_PROPERTY_FLAG 0 /* No PAC */
 #endif
 
+/* straight-line speculation mitigation */
+#define SPEC_STOP dsb sy; isb
+
 #ifdef HAVE_GCC_ASM_CFI_DIRECTIVES
 /* CFI directives to emit DWARF stack unwinding information. */
 # define CFI_STARTPROC()            .cfi_startproc; AARCH64_HINT_BTI_C
-# define CFI_ENDPROC()              .cfi_endproc
+# define CFI_ENDPROC()              SPEC_STOP; .cfi_endproc
 # define CFI_REMEMBER_STATE()       .cfi_remember_state
 # define CFI_RESTORE_STATE()        .cfi_restore_state
 # define CFI_ADJUST_CFA_OFFSET(off) .cfi_adjust_cfa_offset off
@@ -106,7 +109,7 @@
 
 #else
 # define CFI_STARTPROC() AARCH64_HINT_BTI_C
-# define CFI_ENDPROC()
+# define CFI_ENDPROC() SPEC_STOP
 # define CFI_REMEMBER_STATE()
 # define CFI_RESTORE_STATE()
 # define CFI_ADJUST_CFA_OFFSET(off)
@@ -121,7 +124,7 @@
 
 /* 'ret' instruction replacement for straight-line speculation mitigation */
 #define ret_spec_stop \
-	ret; dsb sy; isb;
+	ret; SPEC_STOP;
 
 #define CLEAR_REG(reg) movi reg.16b, #0;
 
diff --git a/cipher/asm-common-amd64.h b/cipher/asm-common-amd64.h
index d0cc6426..a827c025 100644
--- a/cipher/asm-common-amd64.h
+++ b/cipher/asm-common-amd64.h
@@ -88,10 +88,13 @@
 #define ENDBRANCH /*_*/
 #endif
 
+/* straight-line speculation mitigation */
+#define SPEC_STOP int3
+
 #ifdef HAVE_GCC_ASM_CFI_DIRECTIVES
 /* CFI directives to emit DWARF stack unwinding information. */
 # define CFI_STARTPROC()            .cfi_startproc; ENDBRANCH
-# define CFI_ENDPROC()              .cfi_endproc
+# define CFI_ENDPROC()              SPEC_STOP; .cfi_endproc
 # define CFI_REMEMBER_STATE()       .cfi_remember_state
 # define CFI_RESTORE_STATE()        .cfi_restore_state
 # define CFI_ADJUST_CFA_OFFSET(off) .cfi_adjust_cfa_offset off
@@ -159,7 +162,7 @@
 
 #else
 # define CFI_STARTPROC() ENDBRANCH
-# define CFI_ENDPROC()
+# define CFI_ENDPROC() SPEC_STOP
 # define CFI_REMEMBER_STATE()
 # define CFI_RESTORE_STATE()
 # define CFI_ADJUST_CFA_OFFSET(off)
@@ -210,7 +213,7 @@
 
 /* 'ret' instruction replacement for straight-line speculation mitigation. */
 #define ret_spec_stop \
-	ret; int3;
+	ret; SPEC_STOP;
 
 /* This prevents speculative execution on old AVX512 CPUs, to prevent
  * speculative execution to AVX512 code. The vpopcntb instruction is
diff --git a/cipher/asm-common-i386.h b/cipher/asm-common-i386.h
index 346a8ff2..9e85a54e 100644
--- a/cipher/asm-common-i386.h
+++ b/cipher/asm-common-i386.h
@@ -65,10 +65,13 @@
 #define ENDBRANCH /*_*/
 #endif
 
+/* straight-line speculation mitigation */
+#define SPEC_STOP int3
+
 #ifdef HAVE_GCC_ASM_CFI_DIRECTIVES
 /* CFI directives to emit DWARF stack unwinding information. */
 # define CFI_STARTPROC()            .cfi_startproc; ENDBRANCH
-# define CFI_ENDPROC()              .cfi_endproc
+# define CFI_ENDPROC()              SPEC_STOP; .cfi_endproc
 # define CFI_REMEMBER_STATE()       .cfi_remember_state
 # define CFI_RESTORE_STATE()        .cfi_restore_state
 # define CFI_ADJUST_CFA_OFFSET(off) .cfi_adjust_cfa_offset off
@@ -128,7 +131,7 @@
 
 #else
 # define CFI_STARTPROC() ENDBRANCH
-# define CFI_ENDPROC()
+# define CFI_ENDPROC() SPEC_STOP
 # define CFI_REMEMBER_STATE()
 # define CFI_RESTORE_STATE()
 # define CFI_ADJUST_CFA_OFFSET(off)
@@ -148,7 +151,7 @@
 
 /* 'ret' instruction replacement for straight-line speculation mitigation. */
 #define ret_spec_stop \
-	ret; int3;
+	ret; SPEC_STOP;
 
 /* This prevents speculative execution on old AVX512 CPUs, to prevent
  * speculative execution to AVX512 code. The vpopcntb instruction is
diff --git a/cipher/cast5-amd64.S b/cipher/cast5-amd64.S
index b8ae8ba0..aba88c16 100644
--- a/cipher/cast5-amd64.S
+++ b/cipher/cast5-amd64.S
@@ -431,8 +431,8 @@ __cast5_dec_blk4:
 	round_dec_last4(1, F4_2, F4_1);
 
 	outbswap_block4(RLR0, RLR1, RLR2, RLR3);
-	CFI_ENDPROC();
 	ret_spec_stop;
+	CFI_ENDPROC();
 ELF(.size __cast5_dec_blk4,.-__cast5_dec_blk4;)
 
 .align 16
diff --git a/mpi/i386/syntax.h b/mpi/i386/syntax.h
index 2383ebd3..acecb8d9 100644
--- a/mpi/i386/syntax.h
+++ b/mpi/i386/syntax.h
@@ -73,6 +73,9 @@
 #define ALIGN(log) .align log,0x90
 #endif
 
+/* straight-line speculation mitigation */
+#define SPEC_STOP int3
+
 /* 'ret' instruction replacement for straight-line speculation mitigation */
 #define ret_spec_stop \
-	ret; int3;
+	ret; SPEC_STOP;
-- 
2.53.0



From jussi.kivilinna at iki.fi  Sun Jun 28 14:37:37 2026
From: jussi.kivilinna at iki.fi (Jussi Kivilinna)
Date: Sun, 28 Jun 2026 15:37:37 +0300
Subject: [PATCH 3/7] sm3: add Intel SM3 extension implementation
In-Reply-To: <20260628123742.1916-1-jussi.kivilinna@iki.fi>
References: <20260628123742.1916-1-jussi.kivilinna@iki.fi>
Message-ID: <20260628123742.1916-3-jussi.kivilinna@iki.fi>

* LICENSES: Add 'cipher/sm3-intel-avx2-amd64.S'.
* cipher/Makefile.am: Add 'sm3-intel-avx2-amd64.S'.
* cipher/sm3-intel-avx2-amd64.S: New.
* cipher/sm3.c (USE_INTEL_SM3): New.
(ASM_FUNC_ABI, ASM_EXTRA_STACK): Define also for USE_INTEL_SM3.
[USE_INTEL_SM3] (_gcry_sm3_transform_intel_avx2)
(do_sm3_transform_intel_avx2): New.
(sm3_init) [USE_INTEL_SM3]: Use Intel SM3 accelerated implementation if
HW feature available.
* configure.ac (gcry_cv_gcc_inline_asm_sm3)
(HAVE_GCC_INLINE_ASM_SM3): New.
(GCRYPT_ASM_DIGESTS) [x86_64]: Add 'sm3-intel-avx2-amd64.lo'.
* doc/gcrypt.texi: Add "intel-sm3" to HW features list.
* src/g10lib.h (HWF_INTEL_SM3): New.
* src/hwf-x86.c (detect_x86_gnuc): Add Intel SM3 detection.
* src/hwfeatures.c (hwflist): Add "intel-sm3".
--

Converted to GAS assembly from the SM3-NI implementation in intel-ipsec-mb.
Uses the Intel SM3 instructions (VSM3MSG1, VSM3MSG2 and VSM3RNDS2).

Tested with Intel SDE (both 'sde -future' instruction mix and SM3 test
vectors), not yet on real hardware.

Signed-off-by: Jussi Kivilinna <jussi.kivilinna at iki.fi>
---
 LICENSES                      |  30 +++++
 cipher/Makefile.am            |   3 +-
 cipher/sm3-intel-avx2-amd64.S | 228 ++++++++++++++++++++++++++++++++++
 cipher/sm3.c                  |  32 ++++-
 configure.ac                  |  25 ++++
 doc/gcrypt.texi               |   1 +
 src/g10lib.h                  |   1 +
 src/hwf-x86.c                 |   4 +
 src/hwfeatures.c              |   1 +
 9 files changed, 323 insertions(+), 2 deletions(-)
 create mode 100644 cipher/sm3-intel-avx2-amd64.S

diff --git a/LICENSES b/LICENSES
index c2fea82d..e61dca9a 100644
--- a/LICENSES
+++ b/LICENSES
@@ -86,6 +86,36 @@ with any binary distributions derived from the GNU C Library.
    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #+end_quote
 
+  For files:
+  - cipher/sm3-intel-avx2-amd64.S
+
+#+begin_quote
+   Copyright (c) 2023-2024, Intel Corporation
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions are met:
+
+       * Redistributions of source code must retain the above copyright notice,
+         this list of conditions and the following disclaimer.
+       * Redistributions in binary form must reproduce the above copyright
+         notice, this list of conditions and the following disclaimer in the
+         documentation and/or other materials provided with the distribution.
+       * Neither the name of Intel Corporation nor the names of its contributors
+         may be used to endorse or promote products derived from this software
+         without specific prior written permission.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+   DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+   FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+   DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+   SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+   CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+   OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#+end_quote
+
   For files:
   - random/jitterentropy-base.c
   - random/jitterentropy-gcd.c
diff --git a/cipher/Makefile.am b/cipher/Makefile.am
index 0abbfea6..b18ccfd9 100644
--- a/cipher/Makefile.am
+++ b/cipher/Makefile.am
@@ -152,7 +152,8 @@ EXTRA_libcipher_la_SOURCES = \
 	sha512-intel-shaext.c \
 	sha512-armv7-neon.S sha512-armv8-aarch64-ce.S sha512-arm.S \
 	sha512-ppc.c sha512-riscv-zvknhb-zvkb.c sha512-ssse3-i386.c \
-	sm3.c sm3-avx-bmi2-amd64.S sm3-aarch64.S sm3-armv8-aarch64-ce.S \
+	sm3.c sm3-avx-bmi2-amd64.S sm3-intel-avx2-amd64.S sm3-aarch64.S \
+	sm3-armv8-aarch64-ce.S \
 	keccak.c keccak_permute_32.h keccak_permute_64.h \
 	keccak-armv7-neon.S keccak-amd64-avx512.S \
 	stribog.c \
diff --git a/cipher/sm3-intel-avx2-amd64.S b/cipher/sm3-intel-avx2-amd64.S
new file mode 100644
index 00000000..62436e93
--- /dev/null
+++ b/cipher/sm3-intel-avx2-amd64.S
@@ -0,0 +1,228 @@
+/*
+;;
+;; Copyright (c) 2023-2024, Intel Corporation
+;;
+;; Redistribution and use in source and binary forms, with or without
+;; modification, are permitted provided that the following conditions are met:
+;;
+;;     * Redistributions of source code must retain the above copyright notice,
+;;       this list of conditions and the following disclaimer.
+;;     * Redistributions in binary form must reproduce the above copyright
+;;       notice, this list of conditions and the following disclaimer in the
+;;       documentation and/or other materials provided with the distribution.
+;;     * Neither the name of Intel Corporation nor the names of its contributors
+;;       may be used to endorse or promote products derived from this software
+;;       without specific prior written permission.
+;;
+;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+;;
+*/
+/*
+ * From:
+ *  https://github.com/intel/intel-ipsec-mb/blob/2b8d6041780fe94b749c98c08b92e7ea06cf50e2/lib/avx2_t4/sm3_ni_x1_avx2.asm
+ *
+ * Conversion to GAS assembly and integration to libgcrypt
+ *  by Jussi Kivilinna <jussi.kivilinna at iki.fi>
+ *
+ * Uses the Intel SM3 instruction set extension (VSM3MSG1/VSM3MSG2/VSM3RNDS2)
+ */
+
+#ifdef __x86_64
+#include <config.h>
+#if (defined(HAVE_COMPATIBLE_GCC_AMD64_PLATFORM_AS) || \
+     defined(HAVE_COMPATIBLE_GCC_WIN64_PLATFORM_AS)) && \
+    defined(HAVE_INTEL_SYNTAX_PLATFORM_AS) && \
+    defined(HAVE_GCC_INLINE_ASM_AVX2) && \
+    defined(HAVE_GCC_INLINE_ASM_SM3) && defined(USE_SM3)
+
+#include "asm-common-amd64.h"
+
+.intel_syntax noprefix
+
+#define arg_hash      rdi
+#define arg_msg       rsi
+#define arg_num_blks  rdx
+
+SECTION_RODATA
+
+ELF(.type _gcry_sm3_intel_avx2_consts, at object)
+_gcry_sm3_intel_avx2_consts:
+
+.align 16
+.LSHUFF_MASK:
+        .byte 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12
+ELF(.size _gcry_sm3_intel_avx2_consts,.-_gcry_sm3_intel_avx2_consts)
+
+.text
+
+/* Create 4 x 32-bit new words of message schedule W[] using SM3-NI ISA. */
+#define SM3MSG(W03_00, W07_04, W11_08, W15_12, W19_16, T1, T2) \
+        vpalignr W19_16, W11_08, W07_04, 3*4;  /* W19_16 = W10 W9 W8 W7 */ \
+        vpsrldq  T1, W15_12, 4;                 /* T1 = 0 W15 W14 W13 */ \
+        vsm3msg1 W19_16, T1, W03_00;            /* W19_16 = WTMP3..0 */ \
+        vpalignr T1, W07_04, W03_00, 3*4;       /* T1 = W6 W5 W4 W3 */ \
+        vpalignr T2, W15_12, W11_08, 2*4;       /* T2 = W13 W12 W11 W10 */ \
+        vsm3msg2 W19_16, T1, T2;                /* W19_16 = W19 W18 W17 W16 */
+
+/* Perform 4 rounds of SM3, consuming 4 words of message schedule W[] and
+ * updating the SM3 state registers ABEF and CDGH. */
+#define SM3ROUNDS4(ABEF, CDGH, W03_00, W07_04, T1, R) \
+        vpunpcklqdq T1, W03_00, W07_04;         /* T1 = W5 W4 W1 W0 */ \
+        vsm3rnds2   CDGH, ABEF, T1, R;          /* CDGH = updated ABEF */ \
+        vpunpckhqdq T1, W03_00, W07_04;         /* T1 = W7 W6 W3 W2 */ \
+        vsm3rnds2   ABEF, CDGH, T1, ((R) + 2);  /* ABEF = updated CDGH */
+
+/*
+ * unsigned int
+ * _gcry_sm3_transform_intel_avx2 (void *state, const unsigned char *data,
+ *                                 size_t nblks)
+ */
+.align 16
+.globl _gcry_sm3_transform_intel_avx2
+ELF(.type _gcry_sm3_transform_intel_avx2, at function)
+_gcry_sm3_transform_intel_avx2:
+        /* input:
+         *   %rdi: state (8 x u32 hash value)
+         *   %rsi: data (message blocks)
+         *   %rdx: nblks
+         */
+        CFI_STARTPROC();
+
+        test    arg_num_blks, arg_num_blks;
+        jz      .Ldone;
+
+        /* Load current hash value and change word order. */
+        vmovdqu xmm6, [arg_hash + 0*16];        /* xmm6 = D C B A */
+        vmovdqu xmm7, [arg_hash + 1*16];        /* xmm7 = H G F E */
+        vpshufd xmm0, xmm6, 0x1B;               /* xmm0 = A B C D */
+        vpshufd xmm1, xmm7, 0x1B;               /* xmm1 = E F G H */
+        vpunpckhqdq xmm6, xmm1, xmm0;           /* xmm6 = A B E F */
+        vpunpcklqdq xmm7, xmm1, xmm0;           /* xmm7 = C D G H */
+
+        /* Pre-rotate C,D,G,H. */
+        vpsrld  xmm2, xmm7, 9;
+        vpslld  xmm3, xmm7, 23;
+        vpxor   xmm1, xmm2, xmm3;               /* xmm1 = ROL32(CDGH, 23) */
+        vpsrld  xmm4, xmm7, 19;
+        vpslld  xmm5, xmm7, 13;
+        vpxor   xmm0, xmm4, xmm5;               /* xmm0 = ROL32(CDGH, 13) */
+        vpblendd xmm7, xmm1, xmm0, 0x3;         /* xmm7 = ROL(C,23) ROL(D,23)
+                                                          ROL(G,13) ROL(H,13) */
+
+        vmovdqa xmm12, [.LSHUFF_MASK ADD_RIP];
+
+.align 16
+.Lblock_loop:
+        vmovdqa xmm10, xmm6;
+        vmovdqa xmm11, xmm7;
+
+        /* Prepare W[0..15] - read and byte-swap the message words. */
+        vmovdqu xmm2, [arg_msg + 0*16];
+        vmovdqu xmm3, [arg_msg + 1*16];
+        vmovdqu xmm4, [arg_msg + 2*16];
+        vmovdqu xmm5, [arg_msg + 3*16];
+        vpshufb xmm2, xmm2, xmm12;              /* xmm2 = W03 W02 W01 W00 */
+        vpshufb xmm3, xmm3, xmm12;              /* xmm3 = W07 W06 W05 W04 */
+        vpshufb xmm4, xmm4, xmm12;              /* xmm4 = W11 W10 W09 W08 */
+        vpshufb xmm5, xmm5, xmm12;              /* xmm5 = W15 W14 W13 W12 */
+
+        SM3MSG(xmm2, xmm3, xmm4, xmm5, xmm8, xmm9, xmm1)        /* W19..16 */
+        SM3ROUNDS4(xmm6, xmm7, xmm2, xmm3, xmm1, 0)
+
+        vmovdqa xmm2, xmm8;
+        SM3MSG(xmm3, xmm4, xmm5, xmm2, xmm8, xmm9, xmm1)        /* W23..20 */
+        SM3ROUNDS4(xmm6, xmm7, xmm3, xmm4, xmm1, 4)
+
+        vmovdqa xmm3, xmm8;
+        SM3MSG(xmm4, xmm5, xmm2, xmm3, xmm8, xmm9, xmm1)        /* W27..24 */
+        SM3ROUNDS4(xmm6, xmm7, xmm4, xmm5, xmm1, 8)
+
+        vmovdqa xmm4, xmm8;
+        SM3MSG(xmm5, xmm2, xmm3, xmm4, xmm8, xmm9, xmm1)        /* W31..28 */
+        SM3ROUNDS4(xmm6, xmm7, xmm5, xmm2, xmm1, 12)
+
+        vmovdqa xmm5, xmm8;
+        SM3MSG(xmm2, xmm3, xmm4, xmm5, xmm8, xmm9, xmm1)        /* W35..32 */
+        SM3ROUNDS4(xmm6, xmm7, xmm2, xmm3, xmm1, 16)
+
+        vmovdqa xmm2, xmm8;
+        SM3MSG(xmm3, xmm4, xmm5, xmm2, xmm8, xmm9, xmm1)        /* W39..36 */
+        SM3ROUNDS4(xmm6, xmm7, xmm3, xmm4, xmm1, 20)
+
+        vmovdqa xmm3, xmm8;
+        SM3MSG(xmm4, xmm5, xmm2, xmm3, xmm8, xmm9, xmm1)        /* W43..40 */
+        SM3ROUNDS4(xmm6, xmm7, xmm4, xmm5, xmm1, 24)
+
+        vmovdqa xmm4, xmm8;
+        SM3MSG(xmm5, xmm2, xmm3, xmm4, xmm8, xmm9, xmm1)        /* W47..44 */
+        SM3ROUNDS4(xmm6, xmm7, xmm5, xmm2, xmm1, 28)
+
+        vmovdqa xmm5, xmm8;
+        SM3MSG(xmm2, xmm3, xmm4, xmm5, xmm8, xmm9, xmm1)        /* W51..48 */
+        SM3ROUNDS4(xmm6, xmm7, xmm2, xmm3, xmm1, 32)
+
+        vmovdqa xmm2, xmm8;
+        SM3MSG(xmm3, xmm4, xmm5, xmm2, xmm8, xmm9, xmm1)        /* W55..52 */
+        SM3ROUNDS4(xmm6, xmm7, xmm3, xmm4, xmm1, 36)
+
+        vmovdqa xmm3, xmm8;
+        SM3MSG(xmm4, xmm5, xmm2, xmm3, xmm8, xmm9, xmm1)        /* W59..56 */
+        SM3ROUNDS4(xmm6, xmm7, xmm4, xmm5, xmm1, 40)
+
+        vmovdqa xmm4, xmm8;
+        SM3MSG(xmm5, xmm2, xmm3, xmm4, xmm8, xmm9, xmm1)        /* W63..60 */
+        SM3ROUNDS4(xmm6, xmm7, xmm5, xmm2, xmm1, 44)
+
+        vmovdqa xmm5, xmm8;
+        SM3MSG(xmm2, xmm3, xmm4, xmm5, xmm8, xmm9, xmm1)        /* W67..64 */
+        SM3ROUNDS4(xmm6, xmm7, xmm2, xmm3, xmm1, 48)
+
+        vmovdqa xmm2, xmm8;
+        SM3ROUNDS4(xmm6, xmm7, xmm3, xmm4, xmm1, 52)
+
+        SM3ROUNDS4(xmm6, xmm7, xmm4, xmm5, xmm1, 56)
+
+        SM3ROUNDS4(xmm6, xmm7, xmm5, xmm2, xmm1, 60)
+
+        /* Add feed-forward to the chaining value and move to the next block. */
+        vpxor   xmm6, xmm6, xmm10;
+        vpxor   xmm7, xmm7, xmm11;
+        add     arg_msg, 64;
+        dec     arg_num_blks;
+        jnz     .Lblock_loop;
+
+        /* Un-rotate C,D,G,H. */
+        vpslld  xmm2, xmm7, 9;
+        vpsrld  xmm3, xmm7, 23;
+        vpxor   xmm1, xmm2, xmm3;               /* xmm1 = ROL32(CDGH, 9) */
+        vpslld  xmm4, xmm7, 19;
+        vpsrld  xmm5, xmm7, 13;
+        vpxor   xmm0, xmm4, xmm5;               /* xmm0 = ROL32(CDGH, 19) */
+        vpblendd xmm7, xmm1, xmm0, 0x3;         /* xmm7 = ROL(C,9) ROL(D,9)
+                                                          ROL(G,19) ROL(H,19) */
+        vpshufd xmm0, xmm6, 0x1B;               /* xmm0 = F E B A */
+        vpshufd xmm1, xmm7, 0x1B;               /* xmm1 = H G D C */
+        vpunpcklqdq xmm6, xmm0, xmm1;           /* xmm6 = D C B A */
+        vpunpckhqdq xmm7, xmm0, xmm1;           /* xmm7 = H G F E */
+        vmovdqu [arg_hash + 0*16], xmm6;
+        vmovdqu [arg_hash + 1*16], xmm7;
+
+        vzeroall;
+
+.Ldone:
+        xor     eax, eax;
+        ret_spec_stop;
+        CFI_ENDPROC();
+ELF(.size _gcry_sm3_transform_intel_avx2,.-_gcry_sm3_transform_intel_avx2;)
+
+#endif
+#endif
diff --git a/cipher/sm3.c b/cipher/sm3.c
index bfe9f4c2..6001afff 100644
--- a/cipher/sm3.c
+++ b/cipher/sm3.c
@@ -56,6 +56,16 @@
 # define USE_AVX_BMI2 1
 #endif
 
+/* USE_INTEL_SM3 indicates whether to compile with Intel SM3 extension code. */
+#undef USE_INTEL_SM3
+#if defined(__x86_64__) && \
+    defined(HAVE_GCC_INLINE_ASM_AVX2) && defined(HAVE_GCC_INLINE_ASM_SM3) && \
+    defined(HAVE_INTEL_SYNTAX_PLATFORM_AS) && \
+    (defined(HAVE_COMPATIBLE_GCC_AMD64_PLATFORM_AS) || \
+     defined(HAVE_COMPATIBLE_GCC_WIN64_PLATFORM_AS))
+# define USE_INTEL_SM3 1
+#endif
+
 /* USE_AARCH64_SIMD indicates whether to enable ARMv8 SIMD assembly
  * code. */
 #undef USE_AARCH64_SIMD
@@ -88,7 +98,7 @@ typedef struct {
  * stack to store XMM6-XMM15 needed on Win64. */
 #undef ASM_FUNC_ABI
 #undef ASM_EXTRA_STACK
-#if defined(USE_AVX_BMI2)
+#if defined(USE_AVX_BMI2) || defined(USE_INTEL_SM3)
 # ifdef HAVE_COMPATIBLE_GCC_WIN64_PLATFORM_AS
 #  define ASM_FUNC_ABI __attribute__((sysv_abi))
 #  define ASM_EXTRA_STACK (10 * 16 + 4 * sizeof(void *))
@@ -115,6 +125,22 @@ do_sm3_transform_amd64_avx_bmi2(void *context, const unsigned char *data,
 }
 #endif /* USE_AVX_BMI2 */
 
+#ifdef USE_INTEL_SM3
+unsigned int _gcry_sm3_transform_intel_avx2(void *state,
+                                            const void *input_data,
+                                            size_t num_blks) ASM_FUNC_ABI;
+
+static unsigned int
+do_sm3_transform_intel_avx2(void *context, const unsigned char *data,
+                            size_t nblks)
+{
+  SM3_CONTEXT *hd = context;
+  unsigned int nburn = _gcry_sm3_transform_intel_avx2 (hd->h, data, nblks);
+  nburn += nburn ? ASM_EXTRA_STACK : 0;
+  return nburn;
+}
+#endif /* USE_INTEL_SM3 */
+
 #ifdef USE_AARCH64_SIMD
 unsigned int _gcry_sm3_transform_aarch64(void *state, const void *input_data,
                                          size_t num_blks);
@@ -173,6 +199,10 @@ sm3_init (void *context, unsigned int flags)
   if ((features & HWF_INTEL_AVX2) && (features & HWF_INTEL_BMI2))
     hd->bctx.bwrite = do_sm3_transform_amd64_avx_bmi2;
 #endif
+#ifdef USE_INTEL_SM3
+  if ((features & HWF_INTEL_SM3) && (features & HWF_INTEL_AVX2))
+    hd->bctx.bwrite = do_sm3_transform_intel_avx2;
+#endif
 #ifdef USE_AARCH64_SIMD
   if (features & HWF_ARM_NEON)
     hd->bctx.bwrite = do_sm3_transform_aarch64;
diff --git a/configure.ac b/configure.ac
index c11bc3b6..b174e518 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1612,6 +1612,30 @@ if test "$gcry_cv_gcc_inline_asm_sha512" = "yes" ; then
 fi
 
 
+#
+# Check whether GCC inline assembler supports Intel SM3 instructions.
+#
+AC_CACHE_CHECK([whether GCC inline assembler supports Intel SM3 instructions],
+       [gcry_cv_gcc_inline_asm_sm3],
+       [if test "$mpi_cpu_arch" != "x86" ||
+           test "$try_asm_modules" != "yes" ; then
+          gcry_cv_gcc_inline_asm_sm3="n/a"
+        else
+          gcry_cv_gcc_inline_asm_sm3=no
+          AC_LINK_IFELSE([AC_LANG_PROGRAM(
+          [[void a(void) {
+              __asm__("vsm3msg1 %%xmm2, %%xmm1, %%xmm3\n\t":::"cc");
+              __asm__("vsm3msg2 %%xmm2, %%xmm1, %%xmm3\n\t":::"cc");
+              __asm__("vsm3rnds2 \$0, %%xmm2, %%xmm1, %%xmm3\n\t":::"cc");
+            }]], [ a(); ] )],
+          [gcry_cv_gcc_inline_asm_sm3=yes])
+        fi])
+if test "$gcry_cv_gcc_inline_asm_sm3" = "yes" ; then
+   AC_DEFINE(HAVE_GCC_INLINE_ASM_SM3,1,
+     [Defined if inline assembler supports Intel SM3 instructions])
+fi
+
+
 #
 # Check whether GCC inline assembler supports SSE4.1 instructions.
 #
@@ -4142,6 +4166,7 @@ if test "$found" = "1" ; then
      x86_64-*-*)
         # Build with the assembly implementation
         GCRYPT_ASM_DIGESTS="$GCRYPT_ASM_DIGESTS sm3-avx-bmi2-amd64.lo"
+        GCRYPT_ASM_DIGESTS="$GCRYPT_ASM_DIGESTS sm3-intel-avx2-amd64.lo"
      ;;
      aarch64-*-*)
         # Build with the assembly implementation
diff --git a/doc/gcrypt.texi b/doc/gcrypt.texi
index 58887b8d..b233cd78 100644
--- a/doc/gcrypt.texi
+++ b/doc/gcrypt.texi
@@ -587,6 +587,7 @@ are
 @item intel-avx512
 @item intel-gfni
 @item intel-sha512
+ at item intel-sm3
 @item arm-neon
 @item arm-aes
 @item arm-sha1
diff --git a/src/g10lib.h b/src/g10lib.h
index 6abc5f5b..d0e64a69 100644
--- a/src/g10lib.h
+++ b/src/g10lib.h
@@ -240,6 +240,7 @@ char **_gcry_strtokenize (const char *string, const char *delim);
 #define HWF_INTEL_AVX512        (1 << 17)
 #define HWF_INTEL_GFNI          (1 << 18)
 #define HWF_INTEL_SHA512        (1 << 19)
+#define HWF_INTEL_SM3           (1 << 20)
 
 #elif defined(HAVE_CPU_ARCH_ARM)
 
diff --git a/src/hwf-x86.c b/src/hwf-x86.c
index e3e144d0..f8c3c948 100644
--- a/src/hwf-x86.c
+++ b/src/hwf-x86.c
@@ -425,6 +425,10 @@ detect_x86_gnuc (
       /* Test bit 0 for Intel SHA512 instructions. */
       if ((intel_feat3 & (1 << 0)) && os_supports_avx_avx2_registers)
         result |= HWF_INTEL_SHA512;
+
+      /* Test bit 1 for Intel SM3 instructions. */
+      if ((intel_feat3 & (1 << 1)) && os_supports_avx_avx2_registers)
+        result |= HWF_INTEL_SM3;
     }
 
   /* Check additional feature flags. */
diff --git a/src/hwfeatures.c b/src/hwfeatures.c
index 4c13ce96..4f9053af 100644
--- a/src/hwfeatures.c
+++ b/src/hwfeatures.c
@@ -85,6 +85,7 @@ static struct
     { HWF_INTEL_AVX512,        "intel-avx512" },
     { HWF_INTEL_GFNI,          "intel-gfni" },
     { HWF_INTEL_SHA512,        "intel-sha512" },
+    { HWF_INTEL_SM3,           "intel-sm3" },
     /* Following removed HW feature strings are kept for API compatibility. */
     { 0,                       "intel-fast-vpgather" },
 #elif defined(HAVE_CPU_ARCH_ARM)
-- 
2.53.0



From jussi.kivilinna at iki.fi  Sun Jun 28 14:37:41 2026
From: jussi.kivilinna at iki.fi (Jussi Kivilinna)
Date: Sun, 28 Jun 2026 15:37:41 +0300
Subject: [PATCH 7/7] sm4: deduplicate OCB bulk dispatch using function pointers
In-Reply-To: <20260628123742.1916-1-jussi.kivilinna@iki.fi>
References: <20260628123742.1916-1-jussi.kivilinna@iki.fi>
Message-ID: <20260628123742.1916-7-jussi.kivilinna@iki.fi>

* cipher/sm4.c (ocb_crypt_fn_t, ocb_auth_fn_t) [ASM_FUNC_ABI]: New.
(_gcry_sm4_aesni_avx_ocb_enc, _gcry_sm4_aesni_avx_ocb_dec)
(_gcry_sm4_aesni_avx_ocb_auth, _gcry_sm4_aesni_avx2_ocb_enc)
(_gcry_sm4_aesni_avx2_ocb_dec, _gcry_sm4_aesni_avx2_ocb_auth)
(_gcry_sm4_intel_avx2_ocb_enc, _gcry_sm4_intel_avx2_ocb_dec)
(_gcry_sm4_intel_avx2_ocb_auth, _gcry_sm4_intel_avx512_ocb_enc_blk32)
(_gcry_sm4_intel_avx512_ocb_dec_blk32, _gcry_sm4_gfni_avx2_ocb_enc)
(_gcry_sm4_gfni_avx2_ocb_dec, _gcry_sm4_gfni_avx2_ocb_auth)
(_gcry_sm4_gfni_avx512_ocb_enc, _gcry_sm4_gfni_avx512_ocb_dec)
(_gcry_sm4_gfni_avx512_ocb_auth, _gcry_sm4_gfni_avx512_ocb_enc_blk32)
(_gcry_sm4_gfni_avx512_ocb_dec_blk32): Make 'Ls' an unsized array
parameter.
(_gcry_sm4_ocb_crypt, _gcry_sm4_ocb_auth) [ASM_FUNC_ABI]: Select the
bulk implementation through a function pointer and share the
8/16/32-block processing loops.
--

The bulk OCB encrypt/decrypt and authentication paths repeated the same
block-chunk processing loop for each implementation. Select the
implementation's bulk function through a function pointer instead, so
the loops are written only once.

Signed-off-by: Jussi Kivilinna <jussi.kivilinna at iki.fi>
---
 cipher/sm4.c | 338 ++++++++++++++++-----------------------------------
 1 file changed, 103 insertions(+), 235 deletions(-)

diff --git a/cipher/sm4.c b/cipher/sm4.c
index 37e1a6c3..3c60dafc 100644
--- a/cipher/sm4.c
+++ b/cipher/sm4.c
@@ -172,6 +172,16 @@ static size_t _gcry_sm4_ocb_auth (gcry_cipher_hd_t c, const void *abuf_arg,
 
 typedef bulk_crypt_fn_t crypt_blk1_16_fn_t;
 
+#ifdef ASM_FUNC_ABI
+typedef void (*ocb_crypt_fn_t)(const u32 *rk_enc, unsigned char *out,
+			       const unsigned char *in, unsigned char *offset,
+			       unsigned char *checksum,
+			       const u64 Ls[]) ASM_FUNC_ABI;
+typedef void (*ocb_auth_fn_t)(const u32 *rk_enc, const unsigned char *abuf,
+			      unsigned char *offset, unsigned char *checksum,
+			      const u64 Ls[]) ASM_FUNC_ABI;
+#endif
+
 typedef struct
 {
   u32 rkey_enc[32];
@@ -1934,79 +1944,71 @@ _gcry_sm4_ocb_crypt (gcry_cipher_hd_t c, void *outbuf_arg,
   unsigned char *outbuf = outbuf_arg;
   const unsigned char *inbuf = inbuf_arg;
   u64 blkn = c->u_mode.ocb.data_nblocks;
+  u32 *rk = encrypt ? ctx->rkey_enc : ctx->rkey_dec;
   int burn_stack_depth = 0;
+#ifdef ASM_FUNC_ABI
+  int bulk_func_available = 0;
+  ocb_crypt_fn_t crypt_blk32 = NULL;
+  ocb_crypt_fn_t crypt_blk16 = NULL;
+  ocb_crypt_fn_t crypt_blk8 = NULL;
 
-#ifdef USE_INTEL_SM4_AVX512
-  if (ctx->use_intel_sm4_avx512)
+#ifdef USE_AESNI_AVX
+  if (ctx->use_aesni_avx)
     {
-      u64 Ls[32];
-      u64 *l;
-
-      if (nblocks >= 32)
-	{
-          l = bulk_ocb_prepare_L_pointers_array_blk32 (c, Ls, blkn);
-
-	  /* Process data in 32 block chunks. */
-	  while (nblocks >= 32)
-	    {
-	      blkn += 32;
-	      *l = (uintptr_t)(void *)ocb_get_l (c, blkn - blkn % 32);
-
-	      if (encrypt)
-		_gcry_sm4_intel_avx512_ocb_enc_blk32 (ctx->rkey_enc, outbuf,
-                                                      inbuf, c->u_iv.iv,
-                                                      c->u_ctr.ctr, Ls);
-	      else
-		_gcry_sm4_intel_avx512_ocb_dec_blk32 (ctx->rkey_dec, outbuf,
-                                                      inbuf, c->u_iv.iv,
-                                                      c->u_ctr.ctr, Ls);
-
-	      nblocks -= 32;
-	      outbuf += 32 * 16;
-	      inbuf += 32 * 16;
-	    }
-	}
+      bulk_func_available = 1;
+      crypt_blk8 = encrypt ? _gcry_sm4_aesni_avx_ocb_enc :
+			     _gcry_sm4_aesni_avx_ocb_dec;
+    }
+#endif
+#ifdef USE_AESNI_AVX2
+  if (ctx->use_aesni_avx2)
+    {
+      bulk_func_available = 1;
+      crypt_blk16 = encrypt ? _gcry_sm4_aesni_avx2_ocb_enc :
+			      _gcry_sm4_aesni_avx2_ocb_dec;
+    }
+#endif
+#ifdef USE_GFNI_AVX2
+  if (ctx->use_gfni_avx2)
+    {
+      bulk_func_available = 1;
+      crypt_blk16 = encrypt ? _gcry_sm4_gfni_avx2_ocb_enc :
+			      _gcry_sm4_gfni_avx2_ocb_dec;
+    }
+#endif
+#ifdef USE_GFNI_AVX512
+  if (ctx->use_gfni_avx512)
+    {
+      bulk_func_available = 1;
+      crypt_blk16 = encrypt ? _gcry_sm4_gfni_avx512_ocb_enc :
+			      _gcry_sm4_gfni_avx512_ocb_dec;
+      crypt_blk32 = encrypt ? _gcry_sm4_gfni_avx512_ocb_enc_blk32 :
+			      _gcry_sm4_gfni_avx512_ocb_dec_blk32;
     }
 #endif
-
 #ifdef USE_INTEL_SM4_AVX2
   if (ctx->use_intel_sm4_avx2)
     {
-      u64 Ls[16];
-      u64 *l;
-
-      if (nblocks >= 16)
-	{
-          l = bulk_ocb_prepare_L_pointers_array_blk16 (c, Ls, blkn);
-
-	  /* Process data in 16 block chunks. */
-	  while (nblocks >= 16)
-	    {
-	      blkn += 16;
-	      *l = (uintptr_t)(void *)ocb_get_l(c, blkn - blkn % 16);
-
-	      if (encrypt)
-		_gcry_sm4_intel_avx2_ocb_enc(ctx->rkey_enc, outbuf, inbuf,
-					     c->u_iv.iv, c->u_ctr.ctr, Ls);
-	      else
-		_gcry_sm4_intel_avx2_ocb_dec(ctx->rkey_dec, outbuf, inbuf,
-					     c->u_iv.iv, c->u_ctr.ctr, Ls);
-
-	      nblocks -= 16;
-	      outbuf += 16 * 16;
-	      inbuf += 16 * 16;
-	    }
-	}
+      bulk_func_available = 1;
+      crypt_blk16 = encrypt ? _gcry_sm4_intel_avx2_ocb_enc :
+			      _gcry_sm4_intel_avx2_ocb_dec;
+    }
+#endif
+#ifdef USE_INTEL_SM4_AVX512
+  if (ctx->use_intel_sm4_avx512)
+    {
+      bulk_func_available = 1;
+      crypt_blk32 = encrypt ? _gcry_sm4_intel_avx512_ocb_enc_blk32 :
+			      _gcry_sm4_intel_avx512_ocb_dec_blk32;
     }
 #endif
 
-#ifdef USE_GFNI_AVX512
-  if (ctx->use_gfni_avx512)
+  if (bulk_func_available)
     {
       u64 Ls[32];
       u64 *l;
 
-      if (nblocks >= 32)
+      if (crypt_blk32 != NULL && nblocks >= 32)
 	{
           l = bulk_ocb_prepare_L_pointers_array_blk32 (c, Ls, blkn);
 
@@ -2016,14 +2018,8 @@ _gcry_sm4_ocb_crypt (gcry_cipher_hd_t c, void *outbuf_arg,
 	      blkn += 32;
 	      *l = (uintptr_t)(void *)ocb_get_l (c, blkn - blkn % 32);
 
-	      if (encrypt)
-		_gcry_sm4_gfni_avx512_ocb_enc_blk32 (ctx->rkey_enc, outbuf,
-                                                     inbuf, c->u_iv.iv,
-                                                     c->u_ctr.ctr, Ls);
-	      else
-		_gcry_sm4_gfni_avx512_ocb_dec_blk32 (ctx->rkey_dec, outbuf,
-                                                     inbuf, c->u_iv.iv,
-                                                     c->u_ctr.ctr, Ls);
+	      crypt_blk32 (rk, outbuf, inbuf, c->u_iv.iv,
+			   c->u_ctr.ctr, Ls);
 
 	      nblocks -= 32;
 	      outbuf += 32 * 16;
@@ -2031,66 +2027,7 @@ _gcry_sm4_ocb_crypt (gcry_cipher_hd_t c, void *outbuf_arg,
 	    }
 	}
 
-      if (nblocks >= 16)
-	{
-          l = bulk_ocb_prepare_L_pointers_array_blk16 (c, Ls, blkn);
-
-	  /* Process data in 16 block chunks. */
-	  blkn += 16;
-	  *l = (uintptr_t)(void *)ocb_get_l(c, blkn - blkn % 16);
-
-	  if (encrypt)
-	    _gcry_sm4_gfni_avx512_ocb_enc(ctx->rkey_enc, outbuf, inbuf,
-					  c->u_iv.iv, c->u_ctr.ctr, Ls);
-	  else
-	    _gcry_sm4_gfni_avx512_ocb_dec(ctx->rkey_dec, outbuf, inbuf,
-					  c->u_iv.iv, c->u_ctr.ctr, Ls);
-
-	  nblocks -= 16;
-	  outbuf += 16 * 16;
-	  inbuf += 16 * 16;
-	}
-    }
-#endif
-
-#ifdef USE_GFNI_AVX2
-  if (ctx->use_gfni_avx2)
-    {
-      u64 Ls[16];
-      u64 *l;
-
-      if (nblocks >= 16)
-	{
-          l = bulk_ocb_prepare_L_pointers_array_blk16 (c, Ls, blkn);
-
-	  /* Process data in 16 block chunks. */
-	  while (nblocks >= 16)
-	    {
-	      blkn += 16;
-	      *l = (uintptr_t)(void *)ocb_get_l(c, blkn - blkn % 16);
-
-	      if (encrypt)
-		_gcry_sm4_gfni_avx2_ocb_enc(ctx->rkey_enc, outbuf, inbuf,
-					    c->u_iv.iv, c->u_ctr.ctr, Ls);
-	      else
-		_gcry_sm4_gfni_avx2_ocb_dec(ctx->rkey_dec, outbuf, inbuf,
-					    c->u_iv.iv, c->u_ctr.ctr, Ls);
-
-	      nblocks -= 16;
-	      outbuf += 16 * 16;
-	      inbuf += 16 * 16;
-	    }
-	}
-    }
-#endif
-
-#ifdef USE_AESNI_AVX2
-  if (ctx->use_aesni_avx2)
-    {
-      u64 Ls[16];
-      u64 *l;
-
-      if (nblocks >= 16)
+      if (crypt_blk16 != NULL && nblocks >= 16)
 	{
           l = bulk_ocb_prepare_L_pointers_array_blk16 (c, Ls, blkn);
 
@@ -2100,28 +2037,16 @@ _gcry_sm4_ocb_crypt (gcry_cipher_hd_t c, void *outbuf_arg,
 	      blkn += 16;
 	      *l = (uintptr_t)(void *)ocb_get_l(c, blkn - blkn % 16);
 
-	      if (encrypt)
-		_gcry_sm4_aesni_avx2_ocb_enc(ctx->rkey_enc, outbuf, inbuf,
-					     c->u_iv.iv, c->u_ctr.ctr, Ls);
-	      else
-		_gcry_sm4_aesni_avx2_ocb_dec(ctx->rkey_dec, outbuf, inbuf,
-					     c->u_iv.iv, c->u_ctr.ctr, Ls);
+	      crypt_blk16 (rk, outbuf, inbuf, c->u_iv.iv,
+			   c->u_ctr.ctr, Ls);
 
 	      nblocks -= 16;
 	      outbuf += 16 * 16;
 	      inbuf += 16 * 16;
 	    }
 	}
-    }
-#endif
-
-#ifdef USE_AESNI_AVX
-  if (ctx->use_aesni_avx)
-    {
-      u64 Ls[8];
-      u64 *l;
 
-      if (nblocks >= 8)
+      if (crypt_blk8 != NULL && nblocks >= 8)
 	{
           l = bulk_ocb_prepare_L_pointers_array_blk8 (c, Ls, blkn);
 
@@ -2131,12 +2056,8 @@ _gcry_sm4_ocb_crypt (gcry_cipher_hd_t c, void *outbuf_arg,
 	      blkn += 8;
 	      *l = (uintptr_t)(void *)ocb_get_l(c, blkn - blkn % 8);
 
-	      if (encrypt)
-		_gcry_sm4_aesni_avx_ocb_enc(ctx->rkey_enc, outbuf, inbuf,
-					    c->u_iv.iv, c->u_ctr.ctr, Ls);
-	      else
-		_gcry_sm4_aesni_avx_ocb_dec(ctx->rkey_dec, outbuf, inbuf,
-					    c->u_iv.iv, c->u_ctr.ctr, Ls);
+	      crypt_blk8 (rk, outbuf, inbuf, c->u_iv.iv,
+			  c->u_ctr.ctr, Ls);
 
 	      nblocks -= 8;
 	      outbuf += 8 * 16;
@@ -2150,7 +2071,6 @@ _gcry_sm4_ocb_crypt (gcry_cipher_hd_t c, void *outbuf_arg,
   if (nblocks)
     {
       crypt_blk1_16_fn_t crypt_blk1_16 = ctx->crypt_blk1_16;
-      u32 *rk = encrypt ? ctx->rkey_enc : ctx->rkey_dec;
       unsigned char tmpbuf[16 * 16];
       unsigned int tmp_used = 16;
       size_t nburn;
@@ -2179,95 +2099,53 @@ _gcry_sm4_ocb_auth (gcry_cipher_hd_t c, const void *abuf_arg, size_t nblocks)
   const unsigned char *abuf = abuf_arg;
   u64 blkn = c->u_mode.ocb.aad_nblocks;
   int burn_stack_depth = 0;
+#ifdef ASM_FUNC_ABI
+  int bulk_func_available = 0;
+  ocb_auth_fn_t auth_blk16 = NULL;
+  ocb_auth_fn_t auth_blk8 = NULL;
 
-#ifdef USE_INTEL_SM4_AVX2
-  if (ctx->use_intel_sm4_avx2)
+#ifdef USE_AESNI_AVX
+  if (ctx->use_aesni_avx)
     {
-      u64 Ls[16];
-      u64 *l;
-
-      if (nblocks >= 16)
-	{
-          l = bulk_ocb_prepare_L_pointers_array_blk16 (c, Ls, blkn);
-
-	  /* Process data in 16 block chunks. */
-	  while (nblocks >= 16)
-	    {
-	      blkn += 16;
-	      *l = (uintptr_t)(void *)ocb_get_l(c, blkn - blkn % 16);
-
-	      _gcry_sm4_intel_avx2_ocb_auth(ctx->rkey_enc, abuf,
-					    c->u_mode.ocb.aad_offset,
-					    c->u_mode.ocb.aad_sum, Ls);
-
-	      nblocks -= 16;
-	      abuf += 16 * 16;
-	    }
-	}
+      bulk_func_available = 1;
+      auth_blk8 = _gcry_sm4_aesni_avx_ocb_auth;
     }
 #endif
-
-#ifdef USE_GFNI_AVX512
-  if (ctx->use_gfni_avx512)
+#ifdef USE_AESNI_AVX2
+  if (ctx->use_aesni_avx2)
     {
-      u64 Ls[16];
-      u64 *l;
-
-      if (nblocks >= 16)
-        {
-          l = bulk_ocb_prepare_L_pointers_array_blk16 (c, Ls, blkn);
-
-          /* Process data in 16 block chunks. */
-          while (nblocks >= 16)
-            {
-              blkn += 16;
-              *l = (uintptr_t)(void *)ocb_get_l (c, blkn - blkn % 16);
-
-              _gcry_sm4_gfni_avx512_ocb_auth (ctx->rkey_enc, abuf,
-                                              c->u_mode.ocb.aad_offset,
-                                              c->u_mode.ocb.aad_sum, Ls);
-
-              nblocks -= 16;
-              abuf += 16 * 16;
-            }
-        }
+      bulk_func_available = 1;
+      auth_blk16 = _gcry_sm4_aesni_avx2_ocb_auth;
     }
 #endif
-
 #ifdef USE_GFNI_AVX2
   if (ctx->use_gfni_avx2)
     {
-      u64 Ls[16];
-      u64 *l;
-
-      if (nblocks >= 16)
-	{
-          l = bulk_ocb_prepare_L_pointers_array_blk16 (c, Ls, blkn);
-
-	  /* Process data in 16 block chunks. */
-	  while (nblocks >= 16)
-	    {
-	      blkn += 16;
-	      *l = (uintptr_t)(void *)ocb_get_l(c, blkn - blkn % 16);
-
-	      _gcry_sm4_gfni_avx2_ocb_auth(ctx->rkey_enc, abuf,
-					   c->u_mode.ocb.aad_offset,
-					   c->u_mode.ocb.aad_sum, Ls);
-
-	      nblocks -= 16;
-	      abuf += 16 * 16;
-	    }
-	}
+      bulk_func_available = 1;
+      auth_blk16 = _gcry_sm4_gfni_avx2_ocb_auth;
+    }
+#endif
+#ifdef USE_GFNI_AVX512
+  if (ctx->use_gfni_avx512)
+    {
+      bulk_func_available = 1;
+      auth_blk16 = _gcry_sm4_gfni_avx512_ocb_auth;
+    }
+#endif
+#ifdef USE_INTEL_SM4_AVX2
+  if (ctx->use_intel_sm4_avx2)
+    {
+      bulk_func_available = 1;
+      auth_blk16 = _gcry_sm4_intel_avx2_ocb_auth;
     }
 #endif
 
-#ifdef USE_AESNI_AVX2
-  if (ctx->use_aesni_avx2)
+  if (bulk_func_available)
     {
       u64 Ls[16];
       u64 *l;
 
-      if (nblocks >= 16)
+      if (auth_blk16 != NULL && nblocks >= 16)
 	{
           l = bulk_ocb_prepare_L_pointers_array_blk16 (c, Ls, blkn);
 
@@ -2277,24 +2155,15 @@ _gcry_sm4_ocb_auth (gcry_cipher_hd_t c, const void *abuf_arg, size_t nblocks)
 	      blkn += 16;
 	      *l = (uintptr_t)(void *)ocb_get_l(c, blkn - blkn % 16);
 
-	      _gcry_sm4_aesni_avx2_ocb_auth(ctx->rkey_enc, abuf,
-					    c->u_mode.ocb.aad_offset,
-					    c->u_mode.ocb.aad_sum, Ls);
+	      auth_blk16(ctx->rkey_enc, abuf, c->u_mode.ocb.aad_offset,
+			 c->u_mode.ocb.aad_sum, Ls);
 
 	      nblocks -= 16;
 	      abuf += 16 * 16;
 	    }
 	}
-    }
-#endif
 
-#ifdef USE_AESNI_AVX
-  if (ctx->use_aesni_avx)
-    {
-      u64 Ls[8];
-      u64 *l;
-
-      if (nblocks >= 8)
+      if (auth_blk8 != NULL && nblocks >= 8)
 	{
           l = bulk_ocb_prepare_L_pointers_array_blk8 (c, Ls, blkn);
 
@@ -2304,9 +2173,8 @@ _gcry_sm4_ocb_auth (gcry_cipher_hd_t c, const void *abuf_arg, size_t nblocks)
 	      blkn += 8;
 	      *l = (uintptr_t)(void *)ocb_get_l(c, blkn - blkn % 8);
 
-	      _gcry_sm4_aesni_avx_ocb_auth(ctx->rkey_enc, abuf,
-					   c->u_mode.ocb.aad_offset,
-					   c->u_mode.ocb.aad_sum, Ls);
+	      auth_blk8(ctx->rkey_enc, abuf, c->u_mode.ocb.aad_offset,
+			c->u_mode.ocb.aad_sum, Ls);
 
 	      nblocks -= 8;
 	      abuf += 8 * 16;
-- 
2.53.0



From jussi.kivilinna at iki.fi  Sun Jun 28 14:37:39 2026
From: jussi.kivilinna at iki.fi (Jussi Kivilinna)
Date: Sun, 28 Jun 2026 15:37:39 +0300
Subject: [PATCH 5/7] sm4: add Intel SM4 instructions accelerated AVX2
 implementation
In-Reply-To: <20260628123742.1916-1-jussi.kivilinna@iki.fi>
References: <20260628123742.1916-1-jussi.kivilinna@iki.fi>
Message-ID: <20260628123742.1916-5-jussi.kivilinna@iki.fi>

* cipher/Makefile.am: Add 'sm4-intel-avx2-amd64.S'.
* cipher/sm4-intel-avx2-amd64.S: New.
* cipher/sm4.c (USE_INTEL_SM4_AVX2): New.
(ASM_FUNC_ABI): Define also for Intel SM4 implementation.
(SM4_context): Add 'use_intel_sm4_avx2'.
(_gcry_sm4_intel_avx2_expand_key, _gcry_sm4_intel_avx2_ctr_enc)
(_gcry_sm4_intel_avx2_cbc_dec, _gcry_sm4_intel_avx2_cfb_dec)
(_gcry_sm4_intel_avx2_ocb_enc, _gcry_sm4_intel_avx2_ocb_dec)
(_gcry_sm4_intel_avx2_ocb_auth, _gcry_sm4_intel_avx2_crypt_blk1_16)
(sm4_intel_avx2_crypt_blk1_16): New.
(sm4_expand_key): Add Intel SM4 code-path.
(sm4_setkey): Use Intel SM4 implementation if supported by CPU.
(sm4_encrypt, sm4_decrypt, sm4_get_crypt_blk1_16_fn)
(_gcry_sm4_ctr_enc, _gcry_sm4_cbc_dec, _gcry_sm4_cfb_dec)
(_gcry_sm4_ocb_crypt, _gcry_sm4_ocb_auth) [USE_INTEL_SM4_AVX2]: Add
Intel SM4 code-path.
* configure.ac (gcry_cv_gcc_inline_asm_sm4, HAVE_GCC_INLINE_ASM_SM4): New.
(GCRYPT_ASM_CIPHERS) [x86_64]: Add 'sm4-intel-avx2-amd64.lo'.
* doc/gcrypt.texi: Add "intel-sm4" to HW features list.
* src/g10lib.h (HWF_INTEL_SM4): New.
* src/hwf-x86.c (detect_x86_gnuc): Add Intel SM4 detection.
* src/hwfeatures.c (hwflist): Add "intel-sm4".
--

Add SM4 implementation using the Intel SM4 instruction set extension:
VSM4KEY4 for the key schedule and VSM4RNDS4 for encryption. It is used
in preference to the GFNI/AVX512 implementation when available.

Tested with Intel SDE; not yet tested on real hardware.

Signed-off-by: Jussi Kivilinna <jussi.kivilinna at iki.fi>
---
 cipher/Makefile.am            |   1 +
 cipher/sm4-intel-avx2-amd64.S | 376 ++++++++++++++++++++++++++++++++++
 cipher/sm4.c                  | 216 ++++++++++++++++++-
 configure.ac                  |  24 +++
 doc/gcrypt.texi               |   1 +
 src/g10lib.h                  |   1 +
 src/hwf-x86.c                 |   5 +
 src/hwfeatures.c              |   1 +
 8 files changed, 624 insertions(+), 1 deletion(-)
 create mode 100644 cipher/sm4-intel-avx2-amd64.S

diff --git a/cipher/Makefile.am b/cipher/Makefile.am
index f8777837..04094a10 100644
--- a/cipher/Makefile.am
+++ b/cipher/Makefile.am
@@ -138,6 +138,7 @@ EXTRA_libcipher_la_SOURCES = \
 	simd-common-aarch64.h simd-common-ppc.h simd-common-riscv.h \
 	sm4.c sm4-aesni-avx-amd64.S sm4-aesni-avx2-amd64.S \
 	sm4-avx2-amd64.h \
+	sm4-intel-avx2-amd64.S \
 	sm4-gfni-avx2-amd64.S sm4-gfni-avx512-amd64.S \
 	sm4-aarch64.S sm4-armv8-aarch64-ce.S sm4-armv9-aarch64-sve-ce.S \
 	sm4-ppc.c \
diff --git a/cipher/sm4-intel-avx2-amd64.S b/cipher/sm4-intel-avx2-amd64.S
new file mode 100644
index 00000000..4c715104
--- /dev/null
+++ b/cipher/sm4-intel-avx2-amd64.S
@@ -0,0 +1,376 @@
+/* sm4-intel-avx2-amd64.S  -  Intel SM4 instructions accelerated AVX2 SM4 cipher
+ *
+ * Copyright (C) 2026 Jussi Kivilinna <jussi.kivilinna at iki.fi>
+ *
+ * This file is part of Libgcrypt.
+ *
+ * Libgcrypt is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * Libgcrypt is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <config.h>
+
+#ifdef __x86_64
+#if (defined(HAVE_COMPATIBLE_GCC_AMD64_PLATFORM_AS) || \
+     defined(HAVE_COMPATIBLE_GCC_WIN64_PLATFORM_AS)) && \
+    defined(ENABLE_AVX2_SUPPORT) && defined(HAVE_GCC_INLINE_ASM_SM4)
+
+#include "asm-common-amd64.h"
+
+/* vector registers */
+#define RA0          %ymm0
+#define RA1          %ymm1
+#define RA2          %ymm2
+#define RA3          %ymm3
+#define RA0x         %xmm0
+#define RA1x         %xmm1
+#define RA2x         %xmm2
+#define RA3x         %xmm3
+
+#define RTMP0        %ymm4
+#define RTMP1        %ymm5
+#define RTMP2        %ymm6
+#define RTMP3        %ymm7
+#define RTMP4        %ymm8
+#define RTMP0x       %xmm4
+#define RTMP1x       %xmm5
+#define RTMP2x       %xmm6
+#define RTMP3x       %xmm7
+#define RTMP4x       %xmm8
+
+#define RB0          %ymm9
+#define RB1          %ymm10
+#define RB2          %ymm11
+#define RB3          %ymm12
+#define RB0x         %xmm9
+#define RB1x         %xmm10
+#define RB2x         %xmm11
+#define RB3x         %xmm12
+
+#define RNOT         %ymm13
+#define RNOTx        %xmm13
+
+SECTION_RODATA
+.align 32
+
+ELF(.type _sm4_intel_avx2_consts, at object)
+_sm4_intel_avx2_consts:
+
+/* For output word byteswap */
+.Lbswap128_mask:
+	.byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
+
+/* For input word byte-swap */
+.Lbswap32_mask:
+	.byte 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12
+
+.text
+
+/**********************************************************************
+  SM4 key schedule with Intel SM4 instructions
+ **********************************************************************/
+
+.align 16
+.globl _gcry_sm4_intel_avx2_expand_key
+ELF(.type   _gcry_sm4_intel_avx2_expand_key, at function;)
+_gcry_sm4_intel_avx2_expand_key:
+	/* input:
+	 *	%rdi: 128-bit master key
+	 *	%rsi: rkey_enc
+	 *	%rdx: rkey_dec
+	 *	%rcx: fk array
+	 *	%r8: ck array
+	 */
+	CFI_STARTPROC();
+
+	vmovdqu (%rdi), RTMP0x;
+	vpshufb .Lbswap32_mask rRIP, RTMP0x, RTMP0x;
+	vpxor (%rcx), RTMP0x, RTMP0x;
+
+	vmovdqu (0*4*4)(%r8), RA0x;
+	vmovdqu (1*4*4)(%r8), RA1x;
+	vmovdqu (2*4*4)(%r8), RA2x;
+	vmovdqu (3*4*4)(%r8), RA3x;
+	vmovdqu (4*4*4)(%r8), RB0x;
+	vmovdqu (5*4*4)(%r8), RB1x;
+	vmovdqu (6*4*4)(%r8), RB2x;
+	vmovdqu (7*4*4)(%r8), RB3x;
+
+#define EXPAND_ROUND4(ck, r) \
+	vsm4key4 ck, RTMP0x, RTMP0x; \
+	vpshufd $0x1b, RTMP0x, RTMP1x; \
+	vmovdqu RTMP0x, (4*4*(r))(%rsi); \
+	vmovdqu RTMP1x, ((32-4)*4)-4*4*(r)(%rdx);
+
+	EXPAND_ROUND4(RA0x, 0);
+	EXPAND_ROUND4(RA1x, 1);
+	EXPAND_ROUND4(RA2x, 2);
+	EXPAND_ROUND4(RA3x, 3);
+	EXPAND_ROUND4(RB0x, 4);
+	EXPAND_ROUND4(RB1x, 5);
+	EXPAND_ROUND4(RB2x, 6);
+	EXPAND_ROUND4(RB3x, 7);
+
+#undef EXPAND_ROUND4
+
+	vzeroall;
+	ret_spec_stop;
+	CFI_ENDPROC();
+ELF(.size _gcry_sm4_intel_avx2_expand_key,.-_gcry_sm4_intel_avx2_expand_key;)
+
+/**********************************************************************
+  16-way SM4 with Intel SM4 instructions
+ **********************************************************************/
+
+.align 16
+ELF(.type   __sm4_intel_crypt_blk16, at function;)
+__sm4_intel_crypt_blk16:
+	/* input:
+	 *	%rdi: round key array, CTX
+	 *	RA0, RA1, RA2, RA3, RB0, RB1, RB2, RB3: sixteen parallel
+	 *						plaintext blocks
+	 * output:
+	 *	RA0, RA1, RA2, RA3, RB0, RB1, RB2, RB3: sixteen parallel
+	 * 						ciphertext blocks
+	 */
+	CFI_STARTPROC();
+
+	vbroadcasti128 .Lbswap32_mask rRIP, RTMP0;
+	vpshufb RTMP0, RA0, RA0;
+	vpshufb RTMP0, RA1, RA1;
+	vpshufb RTMP0, RA2, RA2;
+	vpshufb RTMP0, RA3, RA3;
+	vpshufb RTMP0, RB0, RB0;
+	vpshufb RTMP0, RB1, RB1;
+	vpshufb RTMP0, RB2, RB2;
+	vpshufb RTMP0, RB3, RB3;
+
+#define ROUND4(rkoff) \
+	vbroadcasti128 (rkoff)(%rdi), RTMP1; \
+	vsm4rnds4 RTMP1, RA0, RA0; \
+	vsm4rnds4 RTMP1, RA1, RA1; \
+	vsm4rnds4 RTMP1, RA2, RA2; \
+	vsm4rnds4 RTMP1, RA3, RA3; \
+	vsm4rnds4 RTMP1, RB0, RB0; \
+	vsm4rnds4 RTMP1, RB1, RB1; \
+	vsm4rnds4 RTMP1, RB2, RB2; \
+	vsm4rnds4 RTMP1, RB3, RB3;
+
+	ROUND4(0 * 16);
+	ROUND4(1 * 16);
+	ROUND4(2 * 16);
+	ROUND4(3 * 16);
+	ROUND4(4 * 16);
+	ROUND4(5 * 16);
+	ROUND4(6 * 16);
+	ROUND4(7 * 16);
+
+#undef ROUND4
+
+	vbroadcasti128 .Lbswap128_mask rRIP, RTMP0;
+	vpshufb RTMP0, RA0, RA0;
+	vpshufb RTMP0, RA1, RA1;
+	vpshufb RTMP0, RA2, RA2;
+	vpshufb RTMP0, RA3, RA3;
+	vpshufb RTMP0, RB0, RB0;
+	vpshufb RTMP0, RB1, RB1;
+	vpshufb RTMP0, RB2, RB2;
+	vpshufb RTMP0, RB3, RB3;
+
+	ret_spec_stop;
+	CFI_ENDPROC();
+ELF(.size __sm4_intel_crypt_blk16,.-__sm4_intel_crypt_blk16;)
+
+/* Four rounds (one VSM4RNDS4 group) on 8, 4, 2 or 1 parallel blocks. */
+#define ROUND4_BLK8(rkoff) \
+	vbroadcasti128 (rkoff)(%rdi), RTMP1; \
+	vsm4rnds4 RTMP1, RA0, RA0; \
+	vsm4rnds4 RTMP1, RA1, RA1; \
+	vsm4rnds4 RTMP1, RA2, RA2; \
+	vsm4rnds4 RTMP1, RA3, RA3;
+
+#define ROUND4_BLK4(rkoff) \
+	vbroadcasti128 (rkoff)(%rdi), RTMP1; \
+	vsm4rnds4 RTMP1, RA0, RA0; \
+	vsm4rnds4 RTMP1, RA1, RA1;
+
+#define ROUND4_BLK2(rkoff) \
+	vbroadcasti128 (rkoff)(%rdi), RTMP1; \
+	vsm4rnds4 RTMP1, RA0, RA0;
+
+#define ROUND4_BLK1(rkoff) \
+	vsm4rnds4 (rkoff)(%rdi), RA0x, RA0x;
+
+/**********************************************************************
+  8/4/2/1-way SM4 with Intel SM4 instructions
+ **********************************************************************/
+.align 16
+.globl _gcry_sm4_intel_avx2_crypt_blk1_16
+ELF(.type   _gcry_sm4_intel_avx2_crypt_blk1_16, at function;)
+_gcry_sm4_intel_avx2_crypt_blk1_16:
+	/* input:
+	 *	%rdi: round key array, CTX
+	 *	%rsi: dst (1..16 blocks)
+	 *	%rdx: src (1..16 blocks)
+	 *	%rcx: num blocks (1..16)
+	 */
+	CFI_STARTPROC();
+
+	cmpq $16, %rcx;
+	jne .Lblk1_15_intel;
+
+	/* Sixteen blocks: use the eight-register block function. */
+	vmovdqu (0 * 32)(%rdx), RA0;
+	vmovdqu (1 * 32)(%rdx), RA1;
+	vmovdqu (2 * 32)(%rdx), RA2;
+	vmovdqu (3 * 32)(%rdx), RA3;
+	vmovdqu (4 * 32)(%rdx), RB0;
+	vmovdqu (5 * 32)(%rdx), RB1;
+	vmovdqu (6 * 32)(%rdx), RB2;
+	vmovdqu (7 * 32)(%rdx), RB3;
+	call __sm4_intel_crypt_blk16;
+	vmovdqu RA0, (0 * 32)(%rsi);
+	vmovdqu RA1, (1 * 32)(%rsi);
+	vmovdqu RA2, (2 * 32)(%rsi);
+	vmovdqu RA3, (3 * 32)(%rsi);
+	vmovdqu RB0, (4 * 32)(%rsi);
+	vmovdqu RB1, (5 * 32)(%rsi);
+	vmovdqu RB2, (6 * 32)(%rsi);
+	vmovdqu RB3, (7 * 32)(%rsi);
+	jmp .Lblk1_16_done_intel;
+
+.Lblk1_15_intel:
+	movq %rcx, %rax;
+	vbroadcasti128 .Lbswap32_mask rRIP, RTMP0;
+	vbroadcasti128 .Lbswap128_mask rRIP, RTMP2;
+
+	/* Eight blocks (four YMM registers). */
+	cmpq $8, %rax;
+	jb .Lblk1_16_blk4_intel;
+
+	vmovdqu (0 * 32)(%rdx), RA0;
+	vmovdqu (1 * 32)(%rdx), RA1;
+	vmovdqu (2 * 32)(%rdx), RA2;
+	vmovdqu (3 * 32)(%rdx), RA3;
+	leaq (8 * 16)(%rdx), %rdx;
+	vpshufb RTMP0, RA0, RA0;
+	vpshufb RTMP0, RA1, RA1;
+	vpshufb RTMP0, RA2, RA2;
+	vpshufb RTMP0, RA3, RA3;
+	ROUND4_BLK8(0 * 16);
+	ROUND4_BLK8(1 * 16);
+	ROUND4_BLK8(2 * 16);
+	ROUND4_BLK8(3 * 16);
+	ROUND4_BLK8(4 * 16);
+	ROUND4_BLK8(5 * 16);
+	ROUND4_BLK8(6 * 16);
+	ROUND4_BLK8(7 * 16);
+	vpshufb RTMP2, RA0, RA0;
+	vpshufb RTMP2, RA1, RA1;
+	vpshufb RTMP2, RA2, RA2;
+	vpshufb RTMP2, RA3, RA3;
+	vmovdqu RA0, (0 * 32)(%rsi);
+	vmovdqu RA1, (1 * 32)(%rsi);
+	vmovdqu RA2, (2 * 32)(%rsi);
+	vmovdqu RA3, (3 * 32)(%rsi);
+	leaq (8 * 16)(%rsi), %rsi;
+
+	subq $8, %rax;
+	jz .Lblk1_16_done_intel;
+
+.Lblk1_16_blk4_intel:
+	/* Four blocks (two YMM registers). */
+	cmpq $4, %rax;
+	jb .Lblk1_16_blk2_intel;
+
+	vmovdqu (0 * 32)(%rdx), RA0;
+	vmovdqu (1 * 32)(%rdx), RA1;
+	leaq (4 * 16)(%rdx), %rdx;
+	vpshufb RTMP0, RA0, RA0;
+	vpshufb RTMP0, RA1, RA1;
+	ROUND4_BLK4(0 * 16);
+	ROUND4_BLK4(1 * 16);
+	ROUND4_BLK4(2 * 16);
+	ROUND4_BLK4(3 * 16);
+	ROUND4_BLK4(4 * 16);
+	ROUND4_BLK4(5 * 16);
+	ROUND4_BLK4(6 * 16);
+	ROUND4_BLK4(7 * 16);
+	vpshufb RTMP2, RA0, RA0;
+	vpshufb RTMP2, RA1, RA1;
+	vmovdqu RA0, (0 * 32)(%rsi);
+	vmovdqu RA1, (1 * 32)(%rsi);
+	leaq (4 * 16)(%rsi), %rsi;
+
+	subq $4, %rax;
+	jz .Lblk1_16_done_intel;
+
+.Lblk1_16_blk2_intel:
+	/* Two blocks (one YMM register). */
+	cmpq $2, %rax;
+	jb .Lblk1_16_blk1_intel;
+
+	vmovdqu (0 * 32)(%rdx), RA0;
+	leaq (2 * 16)(%rdx), %rdx;
+	vpshufb RTMP0, RA0, RA0;
+	ROUND4_BLK2(0 * 16);
+	ROUND4_BLK2(1 * 16);
+	ROUND4_BLK2(2 * 16);
+	ROUND4_BLK2(3 * 16);
+	ROUND4_BLK2(4 * 16);
+	ROUND4_BLK2(5 * 16);
+	ROUND4_BLK2(6 * 16);
+	ROUND4_BLK2(7 * 16);
+	vpshufb RTMP2, RA0, RA0;
+	vmovdqu RA0, (0 * 32)(%rsi);
+	leaq (2 * 16)(%rsi), %rsi;
+
+	subq $2, %rax;
+	jz .Lblk1_16_done_intel;
+
+.Lblk1_16_blk1_intel:
+	/* One block (one XMM register). */
+	cmpq $1, %rax;
+	jb .Lblk1_16_done_intel;
+
+	vmovdqu (%rdx), RA0x;
+	vpshufb RTMP0x, RA0x, RA0x;
+	ROUND4_BLK1(0 * 16);
+	ROUND4_BLK1(1 * 16);
+	ROUND4_BLK1(2 * 16);
+	ROUND4_BLK1(3 * 16);
+	ROUND4_BLK1(4 * 16);
+	ROUND4_BLK1(5 * 16);
+	ROUND4_BLK1(6 * 16);
+	ROUND4_BLK1(7 * 16);
+	vpshufb RTMP2x, RA0x, RA0x;
+	vmovdqu RA0x, (%rsi);
+
+.Lblk1_16_done_intel:
+	xorl %eax, %eax;
+	vzeroall;
+	ret_spec_stop;
+	CFI_ENDPROC();
+ELF(.size _gcry_sm4_intel_avx2_crypt_blk1_16,.-_gcry_sm4_intel_avx2_crypt_blk1_16;)
+
+#undef ROUND4_BLK8
+#undef ROUND4_BLK4
+#undef ROUND4_BLK2
+#undef ROUND4_BLK1
+
+#define FUNC_NAME(func) _gcry_sm4_intel_avx2_ ## func
+#define SM4_CRYPT_BLK16 __sm4_intel_crypt_blk16
+#include "sm4-avx2-amd64.h"
+
+#endif /*defined(ENABLE_AVX2_SUPPORT) && defined(HAVE_GCC_INLINE_ASM_SM4)*/
+#endif /*__x86_64*/
diff --git a/cipher/sm4.c b/cipher/sm4.c
index 06b843f8..e397e452 100644
--- a/cipher/sm4.c
+++ b/cipher/sm4.c
@@ -74,11 +74,22 @@
 # endif
 #endif
 
+/* USE_INTEL_SM4_AVX2 indicates whether to compile with Intel SM4 instructions
+ * (VSM4RNDS4) based AVX2 code. */
+#undef USE_INTEL_SM4_AVX2
+#if defined(ENABLE_AVX2_SUPPORT) && defined(HAVE_GCC_INLINE_ASM_SM4)
+# if defined(__x86_64__) && (defined(HAVE_COMPATIBLE_GCC_AMD64_PLATFORM_AS) || \
+     defined(HAVE_COMPATIBLE_GCC_WIN64_PLATFORM_AS))
+#  define USE_INTEL_SM4_AVX2 1
+# endif
+#endif
+
 /* Assembly implementations use SystemV ABI, ABI conversion and additional
  * stack to store XMM6-XMM15 needed on Win64. */
 #undef ASM_FUNC_ABI
 #if defined(USE_AESNI_AVX) || defined(USE_AESNI_AVX2) || \
-    defined(USE_GFNI_AVX2) || defined(USE_GFNI_AVX512)
+    defined(USE_GFNI_AVX2) || defined(USE_GFNI_AVX512) || \
+    defined(USE_INTEL_SM4_AVX2)
 # ifdef HAVE_COMPATIBLE_GCC_WIN64_PLATFORM_AS
 #  define ASM_FUNC_ABI __attribute__((sysv_abi))
 # else
@@ -168,6 +179,9 @@ typedef struct
 #ifdef USE_GFNI_AVX512
   unsigned int use_gfni_avx512:1;
 #endif
+#ifdef USE_INTEL_SM4_AVX2
+  unsigned int use_intel_sm4_avx2:1;
+#endif
 #ifdef USE_AARCH64_SIMD
   unsigned int use_aarch64_simd:1;
 #endif
@@ -356,6 +370,55 @@ sm4_aesni_avx2_crypt_blk1_16(void *rk, byte *out, const byte *in,
 
 #endif /* USE_AESNI_AVX2 */
 
+#ifdef USE_INTEL_SM4_AVX2
+extern void _gcry_sm4_intel_avx2_expand_key(const byte *key, u32 *rk_enc,
+					    u32 *rk_dec, const u32 *fk,
+					    const u32 *ck) ASM_FUNC_ABI;
+
+extern void _gcry_sm4_intel_avx2_ctr_enc(const u32 *rk_enc, byte *out,
+					 const byte *in,
+					 byte *ctr) ASM_FUNC_ABI;
+
+extern void _gcry_sm4_intel_avx2_cbc_dec(const u32 *rk_dec, byte *out,
+					 const byte *in,
+					 byte *iv) ASM_FUNC_ABI;
+
+extern void _gcry_sm4_intel_avx2_cfb_dec(const u32 *rk_enc, byte *out,
+					 const byte *in,
+					 byte *iv) ASM_FUNC_ABI;
+
+extern void _gcry_sm4_intel_avx2_ocb_enc(const u32 *rk_enc,
+					 unsigned char *out,
+					 const unsigned char *in,
+					 unsigned char *offset,
+					 unsigned char *checksum,
+					 const u64 Ls[16]) ASM_FUNC_ABI;
+
+extern void _gcry_sm4_intel_avx2_ocb_dec(const u32 *rk_dec,
+					 unsigned char *out,
+					 const unsigned char *in,
+					 unsigned char *offset,
+					 unsigned char *checksum,
+					 const u64 Ls[16]) ASM_FUNC_ABI;
+
+extern void _gcry_sm4_intel_avx2_ocb_auth(const u32 *rk_enc,
+					  const unsigned char *abuf,
+					  unsigned char *offset,
+					  unsigned char *checksum,
+					  const u64 Ls[16]) ASM_FUNC_ABI;
+
+extern unsigned int
+_gcry_sm4_intel_avx2_crypt_blk1_16(u32 *rk, byte *out, const byte *in,
+				   unsigned int num_blks) ASM_FUNC_ABI;
+
+static inline unsigned int
+sm4_intel_avx2_crypt_blk1_16(void *rk, byte *out, const byte *in,
+			     size_t num_blks)
+{
+  return _gcry_sm4_intel_avx2_crypt_blk1_16(rk, out, in, num_blks);
+}
+#endif /* USE_INTEL_SM4_AVX2 */
+
 #ifdef USE_GFNI_AVX2
 extern void _gcry_sm4_gfni_avx2_expand_key(const byte *key, u32 *rk_enc,
                                            u32 *rk_dec, const u32 *fk,
@@ -700,6 +763,15 @@ sm4_expand_key (SM4_context *ctx, const byte *key)
   u32 rk[4];
   int i;
 
+#ifdef USE_INTEL_SM4_AVX2
+  if (ctx->use_intel_sm4_avx2)
+    {
+      _gcry_sm4_intel_avx2_expand_key (key, ctx->rkey_enc, ctx->rkey_dec,
+                                       fk, ck);
+      return;
+    }
+#endif
+
 #ifdef USE_GFNI_AVX512
   if (ctx->use_gfni_avx512)
     {
@@ -798,6 +870,9 @@ sm4_setkey (void *context, const byte *key, const unsigned keylen,
 #ifdef USE_GFNI_AVX512
   ctx->use_gfni_avx512 = (hwf & HWF_INTEL_GFNI) && (hwf & HWF_INTEL_AVX512);
 #endif
+#ifdef USE_INTEL_SM4_AVX2
+  ctx->use_intel_sm4_avx2 = (hwf & HWF_INTEL_SM4) && (hwf & HWF_INTEL_AVX2);
+#endif
 #ifdef USE_AARCH64_SIMD
   ctx->use_aarch64_simd = !!(hwf & HWF_ARM_NEON);
 #endif
@@ -827,6 +902,26 @@ sm4_setkey (void *context, const byte *key, const unsigned keylen,
     }
 #endif
 
+#ifdef USE_INTEL_SM4_AVX2
+  if (ctx->use_intel_sm4_avx2)
+    {
+      /* Disable AESNI and GFNI implementations when Intel SM4 implementation
+       * is enabled. */
+#ifdef USE_AESNI_AVX
+      ctx->use_aesni_avx = 0;
+#endif
+#ifdef USE_AESNI_AVX2
+      ctx->use_aesni_avx2 = 0;
+#endif
+#ifdef USE_GFNI_AVX2
+      ctx->use_gfni_avx2 = 0;
+#endif
+#ifdef USE_GFNI_AVX512
+      ctx->use_gfni_avx512 = 0;
+#endif
+    }
+#endif
+
   ctx->crypt_blk1_16 = sm4_get_crypt_blk1_16_fn(ctx);
 
   /* Setup bulk encryption routines.  */
@@ -876,6 +971,11 @@ sm4_encrypt (void *context, byte *outbuf, const byte *inbuf)
 {
   SM4_context *ctx = context;
 
+#ifdef USE_INTEL_SM4_AVX2
+  if (ctx->use_intel_sm4_avx2)
+    return sm4_intel_avx2_crypt_blk1_16(ctx->rkey_enc, outbuf, inbuf, 1);
+#endif
+
 #ifdef USE_GFNI_AVX512
   if (ctx->use_gfni_avx512)
     return sm4_gfni_avx512_crypt_blk1_16(ctx->rkey_enc, outbuf, inbuf, 1);
@@ -901,6 +1001,11 @@ sm4_decrypt (void *context, byte *outbuf, const byte *inbuf)
 {
   SM4_context *ctx = context;
 
+#ifdef USE_INTEL_SM4_AVX2
+  if (ctx->use_intel_sm4_avx2)
+    return sm4_intel_avx2_crypt_blk1_16(ctx->rkey_dec, outbuf, inbuf, 1);
+#endif
+
 #ifdef USE_GFNI_AVX512
   if (ctx->use_gfni_avx512)
     return sm4_gfni_avx512_crypt_blk1_16(ctx->rkey_dec, outbuf, inbuf, 1);
@@ -1005,6 +1110,12 @@ sm4_get_crypt_blk1_16_fn(SM4_context *ctx)
 {
   if (0)
     ;
+#ifdef USE_INTEL_SM4_AVX2
+  else if (ctx->use_intel_sm4_avx2)
+    {
+      return &sm4_intel_avx2_crypt_blk1_16;
+    }
+#endif
 #ifdef USE_GFNI_AVX512
   else if (ctx->use_gfni_avx512)
     {
@@ -1077,6 +1188,21 @@ _gcry_sm4_ctr_enc(void *context, unsigned char *ctr,
   const byte *inbuf = inbuf_arg;
   int burn_stack_depth = 0;
 
+#ifdef USE_INTEL_SM4_AVX2
+  if (ctx->use_intel_sm4_avx2)
+    {
+      /* Process data in 16 block chunks. */
+      while (nblocks >= 16)
+        {
+          _gcry_sm4_intel_avx2_ctr_enc(ctx->rkey_enc, outbuf, inbuf, ctr);
+
+          nblocks -= 16;
+          outbuf += 16 * 16;
+          inbuf += 16 * 16;
+        }
+    }
+#endif
+
 #ifdef USE_GFNI_AVX512
   if (ctx->use_gfni_avx512)
     {
@@ -1227,6 +1353,21 @@ _gcry_sm4_cbc_dec(void *context, unsigned char *iv,
   const unsigned char *inbuf = inbuf_arg;
   int burn_stack_depth = 0;
 
+#ifdef USE_INTEL_SM4_AVX2
+  if (ctx->use_intel_sm4_avx2)
+    {
+      /* Process data in 16 block chunks. */
+      while (nblocks >= 16)
+        {
+          _gcry_sm4_intel_avx2_cbc_dec(ctx->rkey_dec, outbuf, inbuf, iv);
+
+          nblocks -= 16;
+          outbuf += 16 * 16;
+          inbuf += 16 * 16;
+        }
+    }
+#endif
+
 #ifdef USE_GFNI_AVX512
   if (ctx->use_gfni_avx512)
     {
@@ -1376,6 +1517,21 @@ _gcry_sm4_cfb_dec(void *context, unsigned char *iv,
   const unsigned char *inbuf = inbuf_arg;
   int burn_stack_depth = 0;
 
+#ifdef USE_INTEL_SM4_AVX2
+  if (ctx->use_intel_sm4_avx2)
+    {
+      /* Process data in 16 block chunks. */
+      while (nblocks >= 16)
+        {
+          _gcry_sm4_intel_avx2_cfb_dec(ctx->rkey_enc, outbuf, inbuf, iv);
+
+          nblocks -= 16;
+          outbuf += 16 * 16;
+          inbuf += 16 * 16;
+        }
+    }
+#endif
+
 #ifdef USE_GFNI_AVX512
   if (ctx->use_gfni_avx512)
     {
@@ -1681,6 +1837,37 @@ _gcry_sm4_ocb_crypt (gcry_cipher_hd_t c, void *outbuf_arg,
   u64 blkn = c->u_mode.ocb.data_nblocks;
   int burn_stack_depth = 0;
 
+#ifdef USE_INTEL_SM4_AVX2
+  if (ctx->use_intel_sm4_avx2)
+    {
+      u64 Ls[16];
+      u64 *l;
+
+      if (nblocks >= 16)
+	{
+          l = bulk_ocb_prepare_L_pointers_array_blk16 (c, Ls, blkn);
+
+	  /* Process data in 16 block chunks. */
+	  while (nblocks >= 16)
+	    {
+	      blkn += 16;
+	      *l = (uintptr_t)(void *)ocb_get_l(c, blkn - blkn % 16);
+
+	      if (encrypt)
+		_gcry_sm4_intel_avx2_ocb_enc(ctx->rkey_enc, outbuf, inbuf,
+					     c->u_iv.iv, c->u_ctr.ctr, Ls);
+	      else
+		_gcry_sm4_intel_avx2_ocb_dec(ctx->rkey_dec, outbuf, inbuf,
+					     c->u_iv.iv, c->u_ctr.ctr, Ls);
+
+	      nblocks -= 16;
+	      outbuf += 16 * 16;
+	      inbuf += 16 * 16;
+	    }
+	}
+    }
+#endif
+
 #ifdef USE_GFNI_AVX512
   if (ctx->use_gfni_avx512)
     {
@@ -1861,6 +2048,33 @@ _gcry_sm4_ocb_auth (gcry_cipher_hd_t c, const void *abuf_arg, size_t nblocks)
   u64 blkn = c->u_mode.ocb.aad_nblocks;
   int burn_stack_depth = 0;
 
+#ifdef USE_INTEL_SM4_AVX2
+  if (ctx->use_intel_sm4_avx2)
+    {
+      u64 Ls[16];
+      u64 *l;
+
+      if (nblocks >= 16)
+	{
+          l = bulk_ocb_prepare_L_pointers_array_blk16 (c, Ls, blkn);
+
+	  /* Process data in 16 block chunks. */
+	  while (nblocks >= 16)
+	    {
+	      blkn += 16;
+	      *l = (uintptr_t)(void *)ocb_get_l(c, blkn - blkn % 16);
+
+	      _gcry_sm4_intel_avx2_ocb_auth(ctx->rkey_enc, abuf,
+					    c->u_mode.ocb.aad_offset,
+					    c->u_mode.ocb.aad_sum, Ls);
+
+	      nblocks -= 16;
+	      abuf += 16 * 16;
+	    }
+	}
+    }
+#endif
+
 #ifdef USE_GFNI_AVX512
   if (ctx->use_gfni_avx512)
     {
diff --git a/configure.ac b/configure.ac
index b174e518..f12b590e 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1636,6 +1636,29 @@ if test "$gcry_cv_gcc_inline_asm_sm3" = "yes" ; then
 fi
 
 
+#
+# Check whether GCC inline assembler supports Intel SM4 instructions.
+#
+AC_CACHE_CHECK([whether GCC inline assembler supports Intel SM4 instructions],
+       [gcry_cv_gcc_inline_asm_sm4],
+       [if test "$mpi_cpu_arch" != "x86" ||
+           test "$try_asm_modules" != "yes" ; then
+          gcry_cv_gcc_inline_asm_sm4="n/a"
+        else
+          gcry_cv_gcc_inline_asm_sm4=no
+          AC_LINK_IFELSE([AC_LANG_PROGRAM(
+          [[void a(void) {
+              __asm__("vsm4key4 %%ymm2, %%ymm1, %%ymm3\n\t":::"cc");
+              __asm__("vsm4rnds4 %%ymm2, %%ymm1, %%ymm3\n\t":::"cc");
+            }]], [ a(); ] )],
+          [gcry_cv_gcc_inline_asm_sm4=yes])
+        fi])
+if test "$gcry_cv_gcc_inline_asm_sm4" = "yes" ; then
+   AC_DEFINE(HAVE_GCC_INLINE_ASM_SM4,1,
+     [Defined if inline assembler supports Intel SM4 instructions])
+fi
+
+
 #
 # Check whether GCC inline assembler supports SSE4.1 instructions.
 #
@@ -3850,6 +3873,7 @@ if test "$found" = "1" ; then
          GCRYPT_ASM_CIPHERS="$GCRYPT_ASM_CIPHERS sm4-aesni-avx2-amd64.lo"
          GCRYPT_ASM_CIPHERS="$GCRYPT_ASM_CIPHERS sm4-gfni-avx2-amd64.lo"
          GCRYPT_ASM_CIPHERS="$GCRYPT_ASM_CIPHERS sm4-gfni-avx512-amd64.lo"
+         GCRYPT_ASM_CIPHERS="$GCRYPT_ASM_CIPHERS sm4-intel-avx2-amd64.lo"
       ;;
       aarch64-*-*)
          # Build with the assembly implementation
diff --git a/doc/gcrypt.texi b/doc/gcrypt.texi
index b233cd78..004c4e30 100644
--- a/doc/gcrypt.texi
+++ b/doc/gcrypt.texi
@@ -588,6 +588,7 @@ are
 @item intel-gfni
 @item intel-sha512
 @item intel-sm3
+ at item intel-sm4
 @item arm-neon
 @item arm-aes
 @item arm-sha1
diff --git a/src/g10lib.h b/src/g10lib.h
index d0e64a69..0b4ca0c7 100644
--- a/src/g10lib.h
+++ b/src/g10lib.h
@@ -241,6 +241,7 @@ char **_gcry_strtokenize (const char *string, const char *delim);
 #define HWF_INTEL_GFNI          (1 << 18)
 #define HWF_INTEL_SHA512        (1 << 19)
 #define HWF_INTEL_SM3           (1 << 20)
+#define HWF_INTEL_SM4           (1 << 21)
 
 #elif defined(HAVE_CPU_ARCH_ARM)
 
diff --git a/src/hwf-x86.c b/src/hwf-x86.c
index f8c3c948..df3ba4f9 100644
--- a/src/hwf-x86.c
+++ b/src/hwf-x86.c
@@ -429,6 +429,11 @@ detect_x86_gnuc (
       /* Test bit 1 for Intel SM3 instructions. */
       if ((intel_feat3 & (1 << 1)) && os_supports_avx_avx2_registers)
         result |= HWF_INTEL_SM3;
+
+      /* Test bit 2 for Intel SM4 instructions.  These instructions are
+       * available for AVX/AVX2/AVX512. */
+      if ((intel_feat3 & (1 << 2)) && os_supports_avx_avx2_registers)
+        result |= HWF_INTEL_SM4;
     }
 
   /* Check additional feature flags. */
diff --git a/src/hwfeatures.c b/src/hwfeatures.c
index 4f9053af..dbc84857 100644
--- a/src/hwfeatures.c
+++ b/src/hwfeatures.c
@@ -86,6 +86,7 @@ static struct
     { HWF_INTEL_GFNI,          "intel-gfni" },
     { HWF_INTEL_SHA512,        "intel-sha512" },
     { HWF_INTEL_SM3,           "intel-sm3" },
+    { HWF_INTEL_SM4,           "intel-sm4" },
     /* Following removed HW feature strings are kept for API compatibility. */
     { 0,                       "intel-fast-vpgather" },
 #elif defined(HAVE_CPU_ARCH_ARM)
-- 
2.53.0



From jussi.kivilinna at iki.fi  Sun Jun 28 14:37:36 2026
From: jussi.kivilinna at iki.fi (Jussi Kivilinna)
Date: Sun, 28 Jun 2026 15:37:36 +0300
Subject: [PATCH 2/7] sha512: add Intel SHA512 extension implementation
In-Reply-To: <20260628123742.1916-1-jussi.kivilinna@iki.fi>
References: <20260628123742.1916-1-jussi.kivilinna@iki.fi>
Message-ID: <20260628123742.1916-2-jussi.kivilinna@iki.fi>

* cipher/Makefile.am: Add 'sha512-intel-shaext.c'; Add instrumentation
option munging for 'sha512-intel-shaext.o' and 'sha512-intel-shaext.lo'.
* cipher/sha512-intel-shaext.c: New.
* cipher/sha512.c (USE_SHA512_INTEL): New.
[USE_SHA512_INTEL] (_gcry_sha512_transform_intel_shaext)
(do_sha512_transform_intel_shaext): New.
(sha512_init_common) [USE_SHA512_INTEL]: Use Intel SHA512 accelerated
implementation if HW feature available.
* configure.ac (gcry_cv_gcc_inline_asm_sha512)
(HAVE_GCC_INLINE_ASM_SHA512): New.
(GCRYPT_ASM_DIGESTS) [x86]: Add 'sha512-intel-shaext.lo'.
* doc/gcrypt.texi: Add "intel-sha512" to HW features list.
* src/g10lib.h (HWF_INTEL_SHA512): New.
* src/hwf-x86.c (get_cpuid_subleaf): New, based on 'get_cpuid'.
(get_cpuid): Implement as wrapper for 'get_cpuid_subleaf'.
(detect_x86_gnuc): Add Intel SHA512 detection.
* src/hwfeatures.c (hwflist): Add "intel-sha512".
--

Implementation has been tested with Intel SDE (both i386 and x86-64),
not yet on real hardware.

Signed-off-by: Jussi Kivilinna <jussi.kivilinna at iki.fi>
---
 cipher/Makefile.am           |   7 +
 cipher/sha512-intel-shaext.c | 298 +++++++++++++++++++++++++++++++++++
 cipher/sha512.c              |  29 ++++
 configure.ac                 |  35 ++++
 doc/gcrypt.texi              |   1 +
 src/g10lib.h                 |   1 +
 src/hwf-x86.c                | 117 ++++++++------
 src/hwfeatures.c             |   1 +
 8 files changed, 445 insertions(+), 44 deletions(-)
 create mode 100644 cipher/sha512-intel-shaext.c

diff --git a/cipher/Makefile.am b/cipher/Makefile.am
index 11bb19d7..0abbfea6 100644
--- a/cipher/Makefile.am
+++ b/cipher/Makefile.am
@@ -149,6 +149,7 @@ EXTRA_libcipher_la_SOURCES = \
 	sha256-intel-shaext.c sha256-ppc.c sha256-riscv-zvknha-zvkb.c \
 	sha512.c sha512-ssse3-amd64.S sha512-avx-amd64.S \
 	sha512-avx2-bmi2-amd64.S sha512-avx512-amd64.S \
+	sha512-intel-shaext.c \
 	sha512-armv7-neon.S sha512-armv8-aarch64-ce.S sha512-arm.S \
 	sha512-ppc.c sha512-riscv-zvknhb-zvkb.c sha512-ssse3-i386.c \
 	sm3.c sm3-avx-bmi2-amd64.S sm3-aarch64.S sm3-armv8-aarch64-ce.S \
@@ -242,6 +243,12 @@ sha256-ssse3-i386.o: $(srcdir)/sha256-ssse3-i386.c Makefile
 sha256-ssse3-i386.lo: $(srcdir)/sha256-ssse3-i386.c Makefile
 	`echo $(LTCOMPILE) -c $< | $(instrumentation_munging) `
 
+sha512-intel-shaext.o: $(srcdir)/sha512-intel-shaext.c Makefile
+	`echo $(COMPILE) -c $< | $(instrumentation_munging) `
+
+sha512-intel-shaext.lo: $(srcdir)/sha512-intel-shaext.c Makefile
+	`echo $(LTCOMPILE) -c $< | $(instrumentation_munging) `
+
 crc-intel-pclmul.o: $(srcdir)/crc-intel-pclmul.c Makefile
 	`echo $(COMPILE) -c $< | $(instrumentation_munging) `
 
diff --git a/cipher/sha512-intel-shaext.c b/cipher/sha512-intel-shaext.c
new file mode 100644
index 00000000..d991abad
--- /dev/null
+++ b/cipher/sha512-intel-shaext.c
@@ -0,0 +1,298 @@
+/* sha512-intel-shaext.c - SHA512 accelerated with Intel SHA512 extension.
+ * Copyright (C) 2026 Jussi Kivilinna <jussi.kivilinna at iki.fi>
+ *
+ * This file is part of Libgcrypt.
+ *
+ * Libgcrypt is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * Libgcrypt is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <config.h>
+
+#include "types.h"
+
+#if defined(HAVE_GCC_INLINE_ASM_SHA512) && \
+    defined(USE_SHA512) && defined(ENABLE_SHAEXT_SUPPORT)
+
+#if _GCRY_GCC_VERSION >= 40400 /* 4.4 */
+/* Prevent compiler from issuing SSE instructions between asm blocks. */
+#  pragma GCC target("no-sse")
+#endif
+#if __clang__
+#  pragma clang attribute push (__attribute__((target("no-sse"))), apply_to = function)
+#endif
+
+#define NO_INSTRUMENT_FUNCTION __attribute__((no_instrument_function))
+
+#define ASM_FUNC_ATTR NO_INSTRUMENT_FUNCTION
+
+/* Two macros to be called prior and after the use of SHA512
+  instructions.  There should be no external function calls between
+  the use of these macros.  There purpose is to make sure that the
+  SSE regsiters are cleared and won't reveal any information about
+  the key or the data.  */
+#ifdef __WIN64__
+/* XMM6-XMM15 are callee-saved registers on WIN64. */
+# define shaext_prepare_variable char win64tmp[2*16]
+# define shaext_prepare_variable_size sizeof(win64tmp)
+# define shaext_prepare()                                               \
+   do { asm volatile ("movdqu %%xmm6, 0*16(%0)\n"                       \
+		      "movdqu %%xmm7, 1*16(%0)\n"                       \
+		      :                                                 \
+		      : "r" (&win64tmp[0])                              \
+		      : "memory");                                      \
+  } while (0)
+# define shaext_cleanup(tmp0,tmp1)                                      \
+   do { asm volatile ("vpxor %%ymm0, %%ymm0, %%ymm0\n\t"                \
+		      "vmovdqu %%ymm0, (%0)\n\t"                        \
+		      "vmovdqu %%ymm0, (%1)\n\t"                        \
+		      "vzeroall\n\t"                                    \
+		      "movdqu 0*16(%2), %%xmm6\n"                       \
+		      "movdqu 1*16(%2), %%xmm7\n"                       \
+		      :                                                 \
+		      : "r" (tmp0), "r" (tmp1), "r" (&win64tmp[0])      \
+		      : "memory");                                      \
+  } while (0)
+#else
+# define shaext_prepare_variable
+# define shaext_prepare_variable_size 0
+# define shaext_prepare() do { } while (0)
+# define shaext_cleanup(tmp0,tmp1)                                      \
+   do { asm volatile ("vpxor %%ymm0, %%ymm0, %%ymm0\n\t"                \
+		      "vmovdqu %%ymm0, (%0)\n\t"                        \
+		      "vmovdqu %%ymm0, (%1)\n\t"                        \
+		      "vzeroall\n\t"                                    \
+		      :                                                 \
+		      : "r" (tmp0), "r" (tmp1)                          \
+		      : "memory");                                      \
+  } while (0)
+#endif
+
+/*
+ * Transform nblks*128 bytes (nblks*16 64-bit words) at DATA.
+ */
+unsigned int ASM_FUNC_ATTR
+_gcry_sha512_transform_intel_shaext(u64 state[8], const unsigned char *data,
+				    size_t nblks, const u64 k[80])
+{
+  static const unsigned char bshuf_mask[16] __attribute__ ((aligned (16))) =
+    { 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8 };
+  char save_buf[2 * 32 + 31];
+  char *abef_save;
+  char *cdgh_save;
+  shaext_prepare_variable;
+
+  if (nblks == 0)
+    return 0;
+
+  shaext_prepare ();
+
+  asm volatile ("" : "=r" (abef_save) : "0" (save_buf) : "memory");
+  abef_save = abef_save + (-(uintptr_t)abef_save & 31);
+  cdgh_save = abef_save + 32;
+
+  /* Load state.  State is {a,b,c,d} and {e,f,g,h} in memory, repack to
+     ABEF (YMM elem order {f,e,b,a}) in YMM1 and CDGH ({h,g,d,c}) in YMM2. */
+  asm volatile ("vpshufd $0x4e, 0*32(%[state]), %%ymm3\n\t" /* {b,a,d,c} */
+		"vpshufd $0x4e, 1*32(%[state]), %%ymm4\n\t" /* {f,e,h,g} */
+		"vperm2i128 $0x02, %%ymm4, %%ymm3, %%ymm1\n\t" /* {f,e,b,a} */
+		"vperm2i128 $0x13, %%ymm4, %%ymm3, %%ymm2\n\t" /* {h,g,d,c} */
+		:
+		: [state] "r" (state)
+		: "memory" );
+
+  /* Load message */
+  asm volatile ("vbroadcasti128 %[mask], %%ymm7\n\t"
+		"vmovdqu 0*32(%[data]), %%ymm3\n\t"
+		"vmovdqu 1*32(%[data]), %%ymm4\n\t"
+		"vmovdqu 2*32(%[data]), %%ymm5\n\t"
+		"vmovdqu 3*32(%[data]), %%ymm6\n\t"
+		"vpshufb %%ymm7, %%ymm3, %%ymm3\n\t"
+		"vpshufb %%ymm7, %%ymm4, %%ymm4\n\t"
+		"vpshufb %%ymm7, %%ymm5, %%ymm5\n\t"
+		"vpshufb %%ymm7, %%ymm6, %%ymm6\n\t"
+		:
+		: [data] "r" (data), [mask] "m" (*bshuf_mask)
+		: "memory" );
+  data += 128;
+
+  do
+    {
+      /* Save state */
+      asm volatile ("vmovdqa %%ymm1, (%[abef_save])\n\t"
+		    "vmovdqa %%ymm2, (%[cdgh_save])\n\t"
+		    :
+		    : [abef_save] "r" (abef_save), [cdgh_save] "r" (cdgh_save)
+		    : "memory" );
+
+
+      /* Rounds 0..3 */
+      asm volatile ("vpaddq 32*0(%[k]), %%ymm3, %%ymm0\n\t"
+		    "vsha512rnds2 %%xmm0, %%ymm1, %%ymm2\n\t"
+		    "vextracti128 $1, %%ymm0, %%xmm0\n\t"
+		    "vsha512rnds2 %%xmm0, %%ymm2, %%ymm1\n\t"
+		    :
+		    : [k] "r" (k)
+		    : "memory" );
+
+      /* Rounds 4..7 */
+      asm volatile ("vpaddq 32*1(%[k]), %%ymm4, %%ymm0\n\t"
+		    "vsha512rnds2 %%xmm0, %%ymm1, %%ymm2\n\t"
+		    "vextracti128 $1, %%ymm0, %%xmm0\n\t"
+		    "vsha512rnds2 %%xmm0, %%ymm2, %%ymm1\n\t"
+		    "vsha512msg1 %%xmm4, %%ymm3\n\t"
+		    :
+		    : [k] "r" (k)
+		    : "memory" );
+
+      /* Rounds 8..11 */
+      asm volatile ("vpaddq 32*2(%[k]), %%ymm5, %%ymm0\n\t"
+		    "vsha512rnds2 %%xmm0, %%ymm1, %%ymm2\n\t"
+		    "vextracti128 $1, %%ymm0, %%xmm0\n\t"
+		    "vsha512rnds2 %%xmm0, %%ymm2, %%ymm1\n\t"
+		    "vsha512msg1 %%xmm5, %%ymm4\n\t"
+		    :
+		    : [k] "r" (k)
+		    : "memory" );
+
+#define ROUND(gr, MSG0, MSG1, MSG2, MSG3) \
+      asm volatile ("vpaddq 32*" #gr "(%[k]), %%ymm" #MSG0 ", %%ymm0\n\t" \
+		    "vsha512rnds2 %%xmm0, %%ymm1, %%ymm2\n\t" \
+		    "vperm2i128 $0x21, %%ymm" #MSG0 ", %%ymm" #MSG3 ", %%ymm7\n\t" \
+		    "vpalignr $8, %%ymm" #MSG3 ", %%ymm7, %%ymm7\n\t" \
+		    "vpaddq %%ymm7, %%ymm" #MSG1 ", %%ymm" #MSG1 "\n\t" \
+		    "vsha512msg2 %%ymm" #MSG0 ", %%ymm" #MSG1 "\n\t" \
+		    "vextracti128 $1, %%ymm0, %%xmm0\n\t" \
+		    "vsha512rnds2 %%xmm0, %%ymm2, %%ymm1\n\t" \
+		    "vsha512msg1 %%xmm" #MSG0 ", %%ymm" #MSG3 "\n\t" \
+		    : \
+		    : [k] "r" (k) \
+		    : "memory" )
+
+      /* Rounds 12..15 to 64..67 (message schedule for W[16..79]). */
+      ROUND(3, 6, 3, 4, 5);
+      ROUND(4, 3, 4, 5, 6);
+      ROUND(5, 4, 5, 6, 3);
+      ROUND(6, 5, 6, 3, 4);
+      ROUND(7, 6, 3, 4, 5);
+      ROUND(8, 3, 4, 5, 6);
+      ROUND(9, 4, 5, 6, 3);
+      ROUND(10, 5, 6, 3, 4);
+      ROUND(11, 6, 3, 4, 5);
+      ROUND(12, 3, 4, 5, 6);
+      ROUND(13, 4, 5, 6, 3);
+      ROUND(14, 5, 6, 3, 4);
+      ROUND(15, 6, 3, 4, 5);
+      ROUND(16, 3, 4, 5, 6);
+
+      if (--nblks == 0)
+	break;
+
+/* Final two message groups: finalize MSG1 but no further message schedule. */
+#define ROUND_FINAL(gr, MSG0, MSG1, MSG3) \
+      asm volatile ("vpaddq 32*" #gr "(%[k]), %%ymm" #MSG0 ", %%ymm0\n\t" \
+		    "vsha512rnds2 %%xmm0, %%ymm1, %%ymm2\n\t" \
+		    "vperm2i128 $0x21, %%ymm" #MSG0 ", %%ymm" #MSG3 ", %%ymm7\n\t" \
+		    "vpalignr $8, %%ymm" #MSG3 ", %%ymm7, %%ymm7\n\t" \
+		    "vpaddq %%ymm7, %%ymm" #MSG1 ", %%ymm" #MSG1 "\n\t" \
+		    "vsha512msg2 %%ymm" #MSG0 ", %%ymm" #MSG1 "\n\t" \
+		    "vextracti128 $1, %%ymm0, %%xmm0\n\t" \
+		    "vsha512rnds2 %%xmm0, %%ymm2, %%ymm1\n\t" \
+		    : \
+		    : [k] "r" (k) \
+		    : "memory" )
+
+      /* Rounds 68..71 */
+      ROUND_FINAL(17, 4, 5, 3);
+
+      asm volatile ("vmovdqu 0*32(%[data]), %%ymm3\n\t"
+		    :
+		    : [data] "r" (data)
+		    : "memory" );
+
+      /* Rounds 72..75 */
+      ROUND_FINAL(18, 5, 6, 4);
+
+      asm volatile ("vbroadcasti128 %[mask], %%ymm7\n\t" /* Reload mask */
+		    "vmovdqu 1*32(%[data]), %%ymm4\n\t"
+		    "vpshufb %%ymm7, %%ymm3, %%ymm3\n\t"
+		    :
+		    : [data] "r" (data), [mask] "m" (*bshuf_mask)
+		    : "memory" );
+
+      /* Rounds 76..79 */
+      asm volatile ("vpaddq 32*19(%[k]), %%ymm6, %%ymm0\n\t"
+		      "vmovdqu 2*32(%[data]), %%ymm5\n\t"
+		    "vsha512rnds2 %%xmm0, %%ymm1, %%ymm2\n\t"
+                      "vmovdqu 3*32(%[data]), %%ymm6\n\t"
+		      "vpshufb %%ymm7, %%ymm4, %%ymm4\n\t"
+		    "vextracti128 $1, %%ymm0, %%xmm0\n\t"
+		      "vpshufb %%ymm7, %%ymm5, %%ymm5\n\t"
+		    "vsha512rnds2 %%xmm0, %%ymm2, %%ymm1\n\t"
+		      "vpshufb %%ymm7, %%ymm6, %%ymm6\n\t"
+		    :
+		    : [k] "r" (k), [data] "r" (data)
+		    : "memory" );
+
+      data += 128;
+
+      /* Merge states */
+      asm volatile ("vpaddq (%[abef_save]), %%ymm1, %%ymm1\n\t"
+		    "vpaddq (%[cdgh_save]), %%ymm2, %%ymm2\n\t"
+		    :
+		    : [abef_save] "r" (abef_save), [cdgh_save] "r" (cdgh_save)
+		    : "memory" );
+    }
+  while (1);
+
+  /* Rounds 68..71 */
+  ROUND_FINAL(17, 4, 5, 3);
+  /* Rounds 72..75 */
+  ROUND_FINAL(18, 5, 6, 4);
+
+  /* Rounds 76..79 */
+  asm volatile ("vpaddq 32*19(%[k]), %%ymm6, %%ymm0\n\t"
+		"vsha512rnds2 %%xmm0, %%ymm1, %%ymm2\n\t"
+		"vextracti128 $1, %%ymm0, %%xmm0\n\t"
+		"vsha512rnds2 %%xmm0, %%ymm2, %%ymm1\n\t"
+		:
+		: [k] "r" (k)
+		: "memory" );
+
+  /* Merge states */
+  asm volatile ("vpaddq (%[abef_save]), %%ymm1, %%ymm1\n\t"
+		"vpaddq (%[cdgh_save]), %%ymm2, %%ymm2\n\t"
+		:
+		: [abef_save] "r" (abef_save), [cdgh_save] "r" (cdgh_save)
+		: "memory" );
+
+  /* Store state.  ABEF=YMM1 ({f,e,b,a}), CDGH=YMM2 ({h,g,d,c}). */
+  asm volatile ("vperm2i128 $0x31, %%ymm2, %%ymm1, %%ymm3\n\t" /* {b,a,d,c} */
+		"vperm2i128 $0x20, %%ymm2, %%ymm1, %%ymm4\n\t" /* {f,e,h,g} */
+		"vpshufd $0x4e, %%ymm3, %%ymm3\n\t"            /* {a,b,c,d} */
+		"vpshufd $0x4e, %%ymm4, %%ymm4\n\t"            /* {e,f,g,h} */
+		"vmovdqu %%ymm3, 0*32(%[state])\n\t"
+		"vmovdqu %%ymm4, 1*32(%[state])\n\t"
+		:
+		: [state] "r" (state)
+		: "memory" );
+
+  shaext_cleanup (abef_save, cdgh_save);
+  return 0;
+}
+
+#if __clang__
+#  pragma clang attribute pop
+#endif
+
+#endif /* HAVE_GCC_INLINE_ASM_SHA512 */
diff --git a/cipher/sha512.c b/cipher/sha512.c
index 51bf6641..024a45ea 100644
--- a/cipher/sha512.c
+++ b/cipher/sha512.c
@@ -133,6 +133,15 @@
 #endif
 
 
+/* USE_SHA512_INTEL indicates whether to compile with Intel SHA512 code. */
+#undef USE_SHA512_INTEL
+#if defined(HAVE_GCC_INLINE_ASM_SHA512) && \
+    defined(USE_SHA512) && \
+    defined(ENABLE_SHAEXT_SUPPORT)
+# define USE_SHA512_INTEL 1
+#endif
+
+
 /* USE_SSSE3_I386 indicates whether to compile with Intel SSSE3/i386 code. */
 #undef USE_SSSE3_I386
 #if defined(__i386__) && SIZEOF_UNSIGNED_LONG == 4 && __GNUC__ >= 4 && \
@@ -345,6 +354,22 @@ do_sha512_transform_amd64_avx512(void *ctx, const unsigned char *data,
 }
 #endif
 
+#ifdef USE_SHA512_INTEL
+/* Does not need ASM_FUNC_ABI */
+unsigned int _gcry_sha512_transform_intel_shaext(u64 state[8],
+                                                 const unsigned char *input_data,
+                                                 size_t num_blks,
+                                                 const u64 k[]);
+
+static unsigned int
+do_sha512_transform_intel_shaext(void *ctx, const unsigned char *data,
+                                 size_t nblks)
+{
+  SHA512_CONTEXT *hd = ctx;
+  return _gcry_sha512_transform_intel_shaext (hd->state.h, data, nblks, k);
+}
+#endif
+
 #ifdef USE_SSSE3_I386
 unsigned int _gcry_sha512_transform_i386_ssse3(u64 state[8],
 					       const unsigned char *input_data,
@@ -508,6 +533,10 @@ sha512_init_common (SHA512_CONTEXT *ctx, unsigned int flags)
   if ((features & HWF_INTEL_SSSE3) != 0)
     ctx->bctx.bwrite = do_sha512_transform_i386_ssse3;
 #endif
+#ifdef USE_SHA512_INTEL
+  if ((features & HWF_INTEL_SHA512) && (features & HWF_INTEL_AVX2))
+    ctx->bctx.bwrite = do_sha512_transform_intel_shaext;
+#endif
 #ifdef USE_RISCV_V_CRYPTO
   if ((features & HWF_RISCV_IMAFDC)
       && (features & HWF_RISCV_B)      /* Mandatory in RVA23U64 */
diff --git a/configure.ac b/configure.ac
index b41f67f4..c11bc3b6 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1584,6 +1584,34 @@ if test "$gcry_cv_gcc_inline_asm_shaext" = "yes" ; then
 fi
 
 
+#
+# Check whether GCC inline assembler supports Intel SHA512 instructions.
+#
+AC_CACHE_CHECK([whether GCC inline assembler supports Intel SHA512 instructions],
+       [gcry_cv_gcc_inline_asm_sha512],
+       [if test "$mpi_cpu_arch" != "x86" ||
+           test "$try_asm_modules" != "yes" ; then
+          gcry_cv_gcc_inline_asm_sha512="n/a"
+        else
+          gcry_cv_gcc_inline_asm_sha512=no
+          AC_LINK_IFELSE([AC_LANG_PROGRAM(
+          [[void a(void) {
+              __asm__("vsha512msg1 %%xmm1, %%ymm3\n\t":::"cc");
+              __asm__("vsha512msg2 %%ymm1, %%ymm3\n\t":::"cc");
+              __asm__("vsha512rnds2 %%xmm0, %%ymm1, %%ymm3\n\t":::"cc");
+              __asm__("vperm2i128 \$0x21, %%ymm1, %%ymm2, %%ymm3\n\t":::"cc");
+              __asm__("vpalignr \$8, %%ymm1, %%ymm2, %%ymm3\n\t":::"cc");
+              __asm__("vpermq \$0x1b, %%ymm1, %%ymm3\n\t":::"cc");
+              __asm__("vextracti128 \$1, %%ymm0, %%xmm1\n\t":::"cc");
+            }]], [ a(); ] )],
+          [gcry_cv_gcc_inline_asm_sha512=yes])
+        fi])
+if test "$gcry_cv_gcc_inline_asm_sha512" = "yes" ; then
+   AC_DEFINE(HAVE_GCC_INLINE_ASM_SHA512,1,
+     [Defined if inline assembler supports Intel SHA512 instructions])
+fi
+
+
 #
 # Check whether GCC inline assembler supports SSE4.1 instructions.
 #
@@ -4042,6 +4070,13 @@ if test "$found" = "1" ; then
          GCRYPT_ASM_DIGESTS="$GCRYPT_ASM_DIGESTS sha512-riscv-zvknhb-zvkb.lo"
       ;;
    esac
+
+   case "$mpi_cpu_arch" in
+     x86)
+       # Build with the Intel SHA512 implementation
+       GCRYPT_ASM_DIGESTS="$GCRYPT_ASM_DIGESTS sha512-intel-shaext.lo"
+     ;;
+   esac
 fi
 
 LIST_MEMBER(sha3, $enabled_digests)
diff --git a/doc/gcrypt.texi b/doc/gcrypt.texi
index e2b8223a..58887b8d 100644
--- a/doc/gcrypt.texi
+++ b/doc/gcrypt.texi
@@ -586,6 +586,7 @@ are
 @item intel-vaes-vpclmul
 @item intel-avx512
 @item intel-gfni
+ at item intel-sha512
 @item arm-neon
 @item arm-aes
 @item arm-sha1
diff --git a/src/g10lib.h b/src/g10lib.h
index bb735e77..6abc5f5b 100644
--- a/src/g10lib.h
+++ b/src/g10lib.h
@@ -239,6 +239,7 @@ char **_gcry_strtokenize (const char *string, const char *delim);
 #define HWF_INTEL_VAES_VPCLMUL  (1 << 16)
 #define HWF_INTEL_AVX512        (1 << 17)
 #define HWF_INTEL_GFNI          (1 << 18)
+#define HWF_INTEL_SHA512        (1 << 19)
 
 #elif defined(HAVE_CPU_ARCH_ARM)
 
diff --git a/src/hwf-x86.c b/src/hwf-x86.c
index e2c9af0c..e3e144d0 100644
--- a/src/hwf-x86.c
+++ b/src/hwf-x86.c
@@ -95,8 +95,9 @@ is_cpuid_available(void)
 }
 
 static void
-get_cpuid(unsigned int in, unsigned int *eax, unsigned int *ebx,
-          unsigned int *ecx, unsigned int *edx)
+get_cpuid_subleaf(unsigned int in, unsigned int subleaf,
+		  unsigned int *eax, unsigned int *ebx,
+		  unsigned int *ecx, unsigned int *edx)
 {
   unsigned int regs[4];
 
@@ -105,7 +106,7 @@ get_cpuid(unsigned int in, unsigned int *eax, unsigned int *ebx,
      "cpuid\n\t"
      "xchgl %%ebx, %1\n\t"     /* Restore GOT register. */
      : "=a" (regs[0]), "=D" (regs[1]), "=c" (regs[2]), "=d" (regs[3])
-     : "0" (in), "1" (0), "2" (0), "3" (0)
+     : "0" (in), "1" (0), "2" (subleaf), "3" (0)
      : "cc"
      );
 
@@ -119,6 +120,13 @@ get_cpuid(unsigned int in, unsigned int *eax, unsigned int *ebx,
     *edx = regs[3];
 }
 
+static void
+get_cpuid(unsigned int in, unsigned int *eax, unsigned int *ebx,
+          unsigned int *ecx, unsigned int *edx)
+{
+  get_cpuid_subleaf(in, 0, eax, ebx, ecx, edx);
+}
+
 #if defined(ENABLE_AVX_SUPPORT) || defined(ENABLE_AVX2_SUPPORT)
 static unsigned int
 get_xgetbv(void)
@@ -148,15 +156,16 @@ is_cpuid_available(void)
 }
 
 static void
-get_cpuid(unsigned int in, unsigned int *eax, unsigned int *ebx,
-          unsigned int *ecx, unsigned int *edx)
+get_cpuid_subleaf(unsigned int in, unsigned int subleaf,
+		  unsigned int *eax, unsigned int *ebx,
+		  unsigned int *ecx, unsigned int *edx)
 {
   unsigned int regs[4];
 
   asm volatile
     ("cpuid\n\t"
      : "=a" (regs[0]), "=b" (regs[1]), "=c" (regs[2]), "=d" (regs[3])
-     : "0" (in), "1" (0), "2" (0), "3" (0)
+     : "0" (in), "1" (0), "2" (subleaf), "3" (0)
      : "cc"
      );
 
@@ -170,6 +179,13 @@ get_cpuid(unsigned int in, unsigned int *eax, unsigned int *ebx,
     *edx = regs[3];
 }
 
+static void
+get_cpuid(unsigned int in, unsigned int *eax, unsigned int *ebx,
+          unsigned int *ecx, unsigned int *edx)
+{
+  get_cpuid_subleaf(in, 0, eax, ebx, ecx, edx);
+}
+
 #if defined(ENABLE_AVX_SUPPORT) || defined(ENABLE_AVX2_SUPPORT)
 static unsigned int
 get_xgetbv(void)
@@ -228,33 +244,35 @@ detect_x86_gnuc (
 #ifdef ENABLE_PADLOCK_SUPPORT
   else if (!strcmp (vendor_id.c, "CentaurHauls"))
     {
+      unsigned int via_feat, via_feat2;
+
       /* This is a VIA CPU.  Check what PadLock features we have.  */
 
       /* Check for extended centaur (EAX).  */
-      get_cpuid(0xC0000000, &features, NULL, NULL, NULL);
+      get_cpuid(0xC0000000, &via_feat, NULL, NULL, NULL);
 
       /* Has extended centaur features? */
-      if (features > 0xC0000000)
+      if (via_feat > 0xC0000000)
         {
            /* Ask for the extended feature flags (EDX). */
-           get_cpuid(0xC0000001, NULL, NULL, NULL, &features);
+           get_cpuid(0xC0000001, NULL, NULL, NULL, &via_feat2);
 
            /* Test bits 2 and 3 to see whether the RNG exists and is enabled. */
-           if ((features & 0x0C) == 0x0C)
+           if ((via_feat2 & 0x0C) == 0x0C)
              result |= HWF_PADLOCK_RNG;
 
            /* Test bits 6 and 7 to see whether the ACE exists and is enabled. */
-           if ((features & 0xC0) == 0xC0)
+           if ((via_feat2 & 0xC0) == 0xC0)
              result |= HWF_PADLOCK_AES;
 
            /* Test bits 10 and 11 to see whether the PHE exists and is
               enabled.  */
-           if ((features & 0xC00) == 0xC00)
+           if ((via_feat2 & 0xC00) == 0xC00)
              result |= HWF_PADLOCK_SHA;
 
            /* Test bits 12 and 13 to see whether the MONTMUL exists and is
               enabled.  */
-           if ((features & 0x3000) == 0x3000)
+           if ((via_feat2 & 0x3000) == 0x3000)
              result |= HWF_PADLOCK_MMUL;
         }
     }
@@ -337,28 +355,30 @@ detect_x86_gnuc (
   has_sse3 = !!(features & 0x00000001);
   if (max_cpuid_level >= 7 && has_sse3)
     {
+      unsigned int intel_feat, intel_feat2, intel_feat3;
+
       /* Get CPUID:7 contains further Intel feature flags. */
-      get_cpuid(7, NULL, &features, &features2, NULL);
+      get_cpuid(7, NULL, &intel_feat, &intel_feat2, NULL);
 
       /* Test bit 8 for BMI2.  */
-      if (features & 0x00000100)
+      if (intel_feat & 0x00000100)
           result |= HWF_INTEL_BMI2;
 
 #ifdef ENABLE_AVX2_SUPPORT
       /* Test bit 5 for AVX2.  */
-      if (features & 0x00000020)
+      if (intel_feat & 0x00000020)
         if (os_supports_avx_avx2_registers)
           result |= HWF_INTEL_AVX2;
 #endif /*ENABLE_AVX_SUPPORT*/
 
       /* Test bit 29 for SHA Extensions. */
-      if (features & (1 << 29))
+      if (intel_feat & (1 << 29))
         result |= HWF_INTEL_SHAEXT;
 
 #if defined(ENABLE_AVX2_SUPPORT) && defined(ENABLE_AESNI_SUPPORT) && \
     defined(ENABLE_PCLMUL_SUPPORT)
-      /* Test features2 bit 9 for VAES and features2 bit 10 for VPCLMULDQD */
-      if ((features2 & 0x00000200) && (features2 & 0x00000400))
+      /* Test intel_feat2 bit 9 for VAES and intel_feat2 bit 10 for VPCLMULDQD */
+      if ((intel_feat2 & 0x00000200) && (intel_feat2 & 0x00000400))
         result |= HWF_INTEL_VAES_VPCLMUL;
 #endif
 
@@ -367,44 +387,53 @@ detect_x86_gnuc (
        * supporting CPUs are new enough not to suffer from reduced clock
        * frequencies when AVX512 is used, which was issue on early AVX512
        * capable CPUs.
-       *  - AVX512F (features bit 16)
-       *  - AVX512DQ (features bit 17)
-       *  - AVX512IFMA (features bit 21)
-       *  - AVX512CD (features bit 28)
-       *  - AVX512BW (features bit 30)
-       *  - AVX512VL (features bit 31)
-       *  - AVX512_VBMI (features2 bit 1)
-       *  - AVX512_VBMI2 (features2 bit 6)
-       *  - AVX512_VNNI (features2 bit 11)
-       *  - AVX512_BITALG (features2 bit 12)
-       *  - AVX512_VPOPCNTDQ (features2 bit 14)
+       *  - AVX512F (intel_feat bit 16)
+       *  - AVX512DQ (intel_feat bit 17)
+       *  - AVX512IFMA (intel_feat bit 21)
+       *  - AVX512CD (intel_feat bit 28)
+       *  - AVX512BW (intel_feat bit 30)
+       *  - AVX512VL (intel_feat bit 31)
+       *  - AVX512_VBMI (intel_feat2 bit 1)
+       *  - AVX512_VBMI2 (intel_feat2 bit 6)
+       *  - AVX512_VNNI (intel_feat2 bit 11)
+       *  - AVX512_BITALG (intel_feat2 bit 12)
+       *  - AVX512_VPOPCNTDQ (intel_feat2 bit 14)
        */
       if (os_supports_avx512_registers
-	  && (features & (1 << 16))
-	  && (features & (1 << 17))
-	  && (features & (1 << 21))
-	  && (features & (1 << 28))
-	  && (features & (1 << 30))
-	  && (features & (1U << 31))
-	  && (features2 & (1 << 1))
-	  && (features2 & (1 << 6))
-	  && (features2 & (1 << 11))
-	  && (features2 & (1 << 12))
-	  && (features2 & (1 << 14)))
+	  && (intel_feat & (1 << 16))
+	  && (intel_feat & (1 << 17))
+	  && (intel_feat & (1 << 21))
+	  && (intel_feat & (1 << 28))
+	  && (intel_feat & (1 << 30))
+	  && (intel_feat & (1U << 31))
+	  && (intel_feat2 & (1 << 1))
+	  && (intel_feat2 & (1 << 6))
+	  && (intel_feat2 & (1 << 11))
+	  && (intel_feat2 & (1 << 12))
+	  && (intel_feat2 & (1 << 14)))
 	result |= HWF_INTEL_AVX512;
 #endif
 
       /* Test features2 bit 6 for GFNI (Galois field new instructions).
        * These instructions are available for SSE/AVX/AVX2/AVX512. */
-      if (features2 & (1 << 6))
+      if (intel_feat2 & (1 << 6))
         result |= HWF_INTEL_GFNI;
+
+      /* Get CPUID:7 sub-leaf 1 for further Intel feature flags. */
+      get_cpuid_subleaf(7, 1, &intel_feat3, NULL, NULL, NULL);
+
+      /* Test bit 0 for Intel SHA512 instructions. */
+      if ((intel_feat3 & (1 << 0)) && os_supports_avx_avx2_registers)
+        result |= HWF_INTEL_SHA512;
     }
 
   /* Check additional feature flags. */
   if (max_cpuid_level >= 0x21 && has_sse3)
     {
-      get_cpuid(0x21, &features, NULL, NULL, NULL);
-      if (features & (1 << 23))
+      unsigned int amd_feat;
+
+      get_cpuid(0x21, &amd_feat, NULL, NULL, NULL);
+      if (amd_feat & (1 << 23))
 	{
 	  has_avx512bmm = 1;
 	}
diff --git a/src/hwfeatures.c b/src/hwfeatures.c
index 7d32c4de..4c13ce96 100644
--- a/src/hwfeatures.c
+++ b/src/hwfeatures.c
@@ -84,6 +84,7 @@ static struct
     { HWF_INTEL_VAES_VPCLMUL,  "intel-vaes-vpclmul" },
     { HWF_INTEL_AVX512,        "intel-avx512" },
     { HWF_INTEL_GFNI,          "intel-gfni" },
+    { HWF_INTEL_SHA512,        "intel-sha512" },
     /* Following removed HW feature strings are kept for API compatibility. */
     { 0,                       "intel-fast-vpgather" },
 #elif defined(HAVE_CPU_ARCH_ARM)
-- 
2.53.0



From jussi.kivilinna at iki.fi  Sun Jun 28 14:37:38 2026
From: jussi.kivilinna at iki.fi (Jussi Kivilinna)
Date: Sun, 28 Jun 2026 15:37:38 +0300
Subject: [PATCH 4/7] sm4: deduplicate AVX2 cipher mode code between AES-NI and
 GFNI
In-Reply-To: <20260628123742.1916-1-jussi.kivilinna@iki.fi>
References: <20260628123742.1916-1-jussi.kivilinna@iki.fi>
Message-ID: <20260628123742.1916-4-jussi.kivilinna@iki.fi>

* cipher/Makefile.am: Add 'sm4-avx2-amd64.h'.
* cipher/sm4-avx2-amd64.h: New, shared cipher mode functions moved
here from 'sm4-aesni-avx2-amd64.S' and 'sm4-gfni-avx2-amd64.S'.
* cipher/sm4-aesni-avx2-amd64.S (__sm4_crypt_blk16): Rename to
'__sm4_aesni_crypt_blk16'.
(_gcry_sm4_aesni_avx2_ctr_enc, _gcry_sm4_aesni_avx2_cbc_dec)
(_gcry_sm4_aesni_avx2_cfb_dec, _gcry_sm4_aesni_avx2_ocb_enc)
(_gcry_sm4_aesni_avx2_ocb_dec, _gcry_sm4_aesni_avx2_ocb_auth)
(inc_le128, Lbige_addb_0_*): Move to 'sm4-avx2-amd64.h'.
(FUNC_NAME, SM4_CRYPT_BLK16): New.
* cipher/sm4-gfni-avx2-amd64.S (_gcry_sm4_gfni_avx2_ctr_enc)
(_gcry_sm4_gfni_avx2_cbc_dec, _gcry_sm4_gfni_avx2_cfb_dec)
(_gcry_sm4_gfni_avx2_ocb_enc, _gcry_sm4_gfni_avx2_ocb_dec)
(_gcry_sm4_gfni_avx2_ocb_auth, inc_le128, Lbige_addb_0_*): Move
to 'sm4-avx2-amd64.h'.
(FUNC_NAME, SM4_CRYPT_BLK16): New.
--

The CTR/CBC/CFB/OCB mode functions were identical between the AES-NI
and GFNI AVX2 implementations apart from the public symbol prefix and
the 16-block transform function called. Move them to a shared header
included by both, parametrized through FUNC_NAME and SM4_CRYPT_BLK16,
as already done for the Camellia AVX2 implementations.

Signed-off-by: Jussi Kivilinna <jussi.kivilinna at iki.fi>
---
 cipher/Makefile.am            |   1 +
 cipher/sm4-aesni-avx2-amd64.S | 630 +--------------------------------
 cipher/sm4-avx2-amd64.h       | 648 ++++++++++++++++++++++++++++++++++
 cipher/sm4-gfni-avx2-amd64.S  | 620 +-------------------------------
 4 files changed, 662 insertions(+), 1237 deletions(-)
 create mode 100644 cipher/sm4-avx2-amd64.h

diff --git a/cipher/Makefile.am b/cipher/Makefile.am
index b18ccfd9..f8777837 100644
--- a/cipher/Makefile.am
+++ b/cipher/Makefile.am
@@ -137,6 +137,7 @@ EXTRA_libcipher_la_SOURCES = \
 	serpent-avx512-x86.c serpent-armv7-neon.S \
 	simd-common-aarch64.h simd-common-ppc.h simd-common-riscv.h \
 	sm4.c sm4-aesni-avx-amd64.S sm4-aesni-avx2-amd64.S \
+	sm4-avx2-amd64.h \
 	sm4-gfni-avx2-amd64.S sm4-gfni-avx512-amd64.S \
 	sm4-aarch64.S sm4-armv8-aarch64-ce.S sm4-armv9-aarch64-sve-ce.S \
 	sm4-ppc.c \
diff --git a/cipher/sm4-aesni-avx2-amd64.S b/cipher/sm4-aesni-avx2-amd64.S
index 03f979fa..128cac80 100644
--- a/cipher/sm4-aesni-avx2-amd64.S
+++ b/cipher/sm4-aesni-avx2-amd64.S
@@ -1,4 +1,4 @@
-/* sm4-avx2-amd64.S  -  AVX2 implementation of SM4 cipher
+/* sm4-aesni-avx2-amd64.S  -  AESNI/AVX2 implementation of SM4 cipher
  *
  * Copyright (C) 2020, 2022-2023 Jussi Kivilinna <jussi.kivilinna at iki.fi>
  *
@@ -163,7 +163,7 @@ _sm4_aesni_avx2_consts:
 	.byte 0x0d, 0x0a, 0x07, 0x00, 0x01, 0x0e, 0x0b, 0x04
 	.byte 0x05, 0x02, 0x0f, 0x08, 0x09, 0x06, 0x03, 0x0c
 
-/* For CTR-mode IV byteswap */
+/* For output word byteswap */
 .Lbswap128_mask:
 	.byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
 
@@ -171,33 +171,6 @@ _sm4_aesni_avx2_consts:
 .Lbswap32_mask:
 	.byte 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12
 
-/* CTR byte addition constants */
-.align 32
-.Lbige_addb_0_1:
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1
-.Lbige_addb_2_3:
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3
-.Lbige_addb_4_5:
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5
-.Lbige_addb_6_7:
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7
-.Lbige_addb_8_9:
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9
-.Lbige_addb_10_11:
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11
-.Lbige_addb_12_13:
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13
-.Lbige_addb_14_15:
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15
-
 .align 4
 /* 4-bit mask */
 .L0f0f0f0f:
@@ -206,8 +179,8 @@ _sm4_aesni_avx2_consts:
 .text
 
 .align 16
-ELF(.type   __sm4_crypt_blk16, at function;)
-__sm4_crypt_blk16:
+ELF(.type   __sm4_aesni_crypt_blk16, at function;)
+__sm4_aesni_crypt_blk16:
 	/* input:
 	 *	%rdi: ctx, CTX
 	 *	RA0, RA1, RA2, RA3, RB0, RB1, RB2, RB3: sixteen parallel
@@ -318,7 +291,7 @@ __sm4_crypt_blk16:
 
 	ret_spec_stop;
 	CFI_ENDPROC();
-ELF(.size __sm4_crypt_blk16,.-__sm4_crypt_blk16;)
+ELF(.size __sm4_aesni_crypt_blk16,.-__sm4_aesni_crypt_blk16;)
 
 .align 16
 .globl _gcry_sm4_aesni_avx2_crypt_blk1_16
@@ -352,7 +325,7 @@ _gcry_sm4_aesni_avx2_crypt_blk1_16:
 #undef LOAD_INPUT
 
 .Lblk16_load_input_done:
-	call __sm4_crypt_blk16;
+	call __sm4_aesni_crypt_blk16;
 
 #define STORE_OUTPUT(yreg, offset) \
 	cmpq $(1 + 2 * (offset)), %rcx; \
@@ -380,594 +353,9 @@ _gcry_sm4_aesni_avx2_crypt_blk1_16:
 	CFI_ENDPROC();
 ELF(.size _gcry_sm4_aesni_avx2_crypt_blk1_16,.-_gcry_sm4_aesni_avx2_crypt_blk1_16;)
 
-#define inc_le128(x, minus_one, tmp) \
-	vpcmpeqq minus_one, x, tmp; \
-	vpsubq minus_one, x, x; \
-	vpslldq $8, tmp, tmp; \
-	vpsubq tmp, x, x;
-
-.align 16
-.globl _gcry_sm4_aesni_avx2_ctr_enc
-ELF(.type   _gcry_sm4_aesni_avx2_ctr_enc, at function;)
-_gcry_sm4_aesni_avx2_ctr_enc:
-	/* input:
-	 *	%rdi: ctx, CTX
-	 *	%rsi: dst (16 blocks)
-	 *	%rdx: src (16 blocks)
-	 *	%rcx: iv (big endian, 128bit)
-	 */
-	CFI_STARTPROC();
-
-	cmpb $(0x100 - 16), 15(%rcx);
-	jbe .Lctr_byteadd;
-
-	movq 8(%rcx), %rax;
-	bswapq %rax;
-
-	vbroadcasti128 .Lbswap128_mask rRIP, RTMP3;
-	vpcmpeqd RNOT, RNOT, RNOT;
-	vpsrldq $8, RNOT, RNOT;   /* ab: -1:0 ; cd: -1:0 */
-	vpaddq RNOT, RNOT, RTMP2; /* ab: -2:0 ; cd: -2:0 */
-
-	/* load IV and byteswap */
-	vmovdqu (%rcx), RTMP4x;
-	vpshufb RTMP3x, RTMP4x, RTMP4x;
-	vmovdqa RTMP4x, RTMP0x;
-	inc_le128(RTMP4x, RNOTx, RTMP1x);
-	vinserti128 $1, RTMP4x, RTMP0, RTMP0;
-	vpshufb RTMP3, RTMP0, RA0; /* +1 ; +0 */
-
-	/* check need for handling 64-bit overflow and carry */
-	cmpq $(0xffffffffffffffff - 16), %rax;
-	ja .Lhandle_ctr_carry;
-
-	/* construct IVs */
-	vpsubq RTMP2, RTMP0, RTMP0; /* +3 ; +2 */
-	vpshufb RTMP3, RTMP0, RA1;
-	vpsubq RTMP2, RTMP0, RTMP0; /* +5 ; +4 */
-	vpshufb RTMP3, RTMP0, RA2;
-	vpsubq RTMP2, RTMP0, RTMP0; /* +7 ; +6 */
-	vpshufb RTMP3, RTMP0, RA3;
-	vpsubq RTMP2, RTMP0, RTMP0; /* +9 ; +8 */
-	vpshufb RTMP3, RTMP0, RB0;
-	vpsubq RTMP2, RTMP0, RTMP0; /* +11 ; +10 */
-	vpshufb RTMP3, RTMP0, RB1;
-	vpsubq RTMP2, RTMP0, RTMP0; /* +13 ; +12 */
-	vpshufb RTMP3, RTMP0, RB2;
-	vpsubq RTMP2, RTMP0, RTMP0; /* +15 ; +14 */
-	vpshufb RTMP3, RTMP0, RB3;
-	vpsubq RTMP2, RTMP0, RTMP0; /* +16 */
-	vpshufb RTMP3x, RTMP0x, RTMP0x;
-
-	jmp .Lctr_carry_done;
-
-.Lhandle_ctr_carry:
-	/* construct IVs */
-	inc_le128(RTMP0, RNOT, RTMP1);
-	inc_le128(RTMP0, RNOT, RTMP1);
-	vpshufb RTMP3, RTMP0, RA1; /* +3 ; +2 */
-	inc_le128(RTMP0, RNOT, RTMP1);
-	inc_le128(RTMP0, RNOT, RTMP1);
-	vpshufb RTMP3, RTMP0, RA2; /* +5 ; +4 */
-	inc_le128(RTMP0, RNOT, RTMP1);
-	inc_le128(RTMP0, RNOT, RTMP1);
-	vpshufb RTMP3, RTMP0, RA3; /* +7 ; +6 */
-	inc_le128(RTMP0, RNOT, RTMP1);
-	inc_le128(RTMP0, RNOT, RTMP1);
-	vpshufb RTMP3, RTMP0, RB0; /* +9 ; +8 */
-	inc_le128(RTMP0, RNOT, RTMP1);
-	inc_le128(RTMP0, RNOT, RTMP1);
-	vpshufb RTMP3, RTMP0, RB1; /* +11 ; +10 */
-	inc_le128(RTMP0, RNOT, RTMP1);
-	inc_le128(RTMP0, RNOT, RTMP1);
-	vpshufb RTMP3, RTMP0, RB2; /* +13 ; +12 */
-	inc_le128(RTMP0, RNOT, RTMP1);
-	inc_le128(RTMP0, RNOT, RTMP1);
-	vpshufb RTMP3, RTMP0, RB3; /* +15 ; +14 */
-	inc_le128(RTMP0, RNOT, RTMP1);
-	vextracti128 $1, RTMP0, RTMP0x;
-	vpshufb RTMP3x, RTMP0x, RTMP0x; /* +16 */
-
-.Lctr_carry_done:
-	/* store new IV */
-	vmovdqu RTMP0x, (%rcx);
-
-.align 8
-.Lload_ctr_done:
-	call __sm4_crypt_blk16;
-
-	vpxor (0 * 32)(%rdx), RA0, RA0;
-	vpxor (1 * 32)(%rdx), RA1, RA1;
-	vpxor (2 * 32)(%rdx), RA2, RA2;
-	vpxor (3 * 32)(%rdx), RA3, RA3;
-	vpxor (4 * 32)(%rdx), RB0, RB0;
-	vpxor (5 * 32)(%rdx), RB1, RB1;
-	vpxor (6 * 32)(%rdx), RB2, RB2;
-	vpxor (7 * 32)(%rdx), RB3, RB3;
-
-	vmovdqu RA0, (0 * 32)(%rsi);
-	vmovdqu RA1, (1 * 32)(%rsi);
-	vmovdqu RA2, (2 * 32)(%rsi);
-	vmovdqu RA3, (3 * 32)(%rsi);
-	vmovdqu RB0, (4 * 32)(%rsi);
-	vmovdqu RB1, (5 * 32)(%rsi);
-	vmovdqu RB2, (6 * 32)(%rsi);
-	vmovdqu RB3, (7 * 32)(%rsi);
-
-	vzeroall;
-
-	ret_spec_stop;
-
-.align 8
-.Lctr_byteadd_full_ctr_carry:
-	movq 8(%rcx), %r11;
-	movq (%rcx), %r10;
-	bswapq %r11;
-	bswapq %r10;
-	addq $16, %r11;
-	adcq $0, %r10;
-	bswapq %r11;
-	bswapq %r10;
-	movq %r11, 8(%rcx);
-	movq %r10, (%rcx);
-	jmp .Lctr_byteadd_ymm;
-.align 8
-.Lctr_byteadd:
-	vbroadcasti128 (%rcx), RB3;
-	je .Lctr_byteadd_full_ctr_carry;
-	addb $16, 15(%rcx);
-.Lctr_byteadd_ymm:
-	vpaddb .Lbige_addb_0_1 rRIP, RB3, RA0;
-	vpaddb .Lbige_addb_2_3 rRIP, RB3, RA1;
-	vpaddb .Lbige_addb_4_5 rRIP, RB3, RA2;
-	vpaddb .Lbige_addb_6_7 rRIP, RB3, RA3;
-	vpaddb .Lbige_addb_8_9 rRIP, RB3, RB0;
-	vpaddb .Lbige_addb_10_11 rRIP, RB3, RB1;
-	vpaddb .Lbige_addb_12_13 rRIP, RB3, RB2;
-	vpaddb .Lbige_addb_14_15 rRIP, RB3, RB3;
-
-	jmp .Lload_ctr_done;
-	CFI_ENDPROC();
-ELF(.size _gcry_sm4_aesni_avx2_ctr_enc,.-_gcry_sm4_aesni_avx2_ctr_enc;)
-
-.align 16
-.globl _gcry_sm4_aesni_avx2_cbc_dec
-ELF(.type   _gcry_sm4_aesni_avx2_cbc_dec, at function;)
-_gcry_sm4_aesni_avx2_cbc_dec:
-	/* input:
-	 *	%rdi: ctx, CTX
-	 *	%rsi: dst (16 blocks)
-	 *	%rdx: src (16 blocks)
-	 *	%rcx: iv
-	 */
-	CFI_STARTPROC();
-
-	vmovdqu (0 * 32)(%rdx), RA0;
-	vmovdqu (1 * 32)(%rdx), RA1;
-	vmovdqu (2 * 32)(%rdx), RA2;
-	vmovdqu (3 * 32)(%rdx), RA3;
-	vmovdqu (4 * 32)(%rdx), RB0;
-	vmovdqu (5 * 32)(%rdx), RB1;
-	vmovdqu (6 * 32)(%rdx), RB2;
-	vmovdqu (7 * 32)(%rdx), RB3;
-
-	call __sm4_crypt_blk16;
-
-	vmovdqu (%rcx), RNOTx;
-	vinserti128 $1, (%rdx), RNOT, RNOT;
-	vpxor RNOT, RA0, RA0;
-	vpxor (0 * 32 + 16)(%rdx), RA1, RA1;
-	vpxor (1 * 32 + 16)(%rdx), RA2, RA2;
-	vpxor (2 * 32 + 16)(%rdx), RA3, RA3;
-	vpxor (3 * 32 + 16)(%rdx), RB0, RB0;
-	vpxor (4 * 32 + 16)(%rdx), RB1, RB1;
-	vpxor (5 * 32 + 16)(%rdx), RB2, RB2;
-	vpxor (6 * 32 + 16)(%rdx), RB3, RB3;
-	vmovdqu (7 * 32 + 16)(%rdx), RNOTx;
-	vmovdqu RNOTx, (%rcx); /* store new IV */
-
-	vmovdqu RA0, (0 * 32)(%rsi);
-	vmovdqu RA1, (1 * 32)(%rsi);
-	vmovdqu RA2, (2 * 32)(%rsi);
-	vmovdqu RA3, (3 * 32)(%rsi);
-	vmovdqu RB0, (4 * 32)(%rsi);
-	vmovdqu RB1, (5 * 32)(%rsi);
-	vmovdqu RB2, (6 * 32)(%rsi);
-	vmovdqu RB3, (7 * 32)(%rsi);
-
-	vzeroall;
-
-	ret_spec_stop;
-	CFI_ENDPROC();
-ELF(.size _gcry_sm4_aesni_avx2_cbc_dec,.-_gcry_sm4_aesni_avx2_cbc_dec;)
-
-.align 16
-.globl _gcry_sm4_aesni_avx2_cfb_dec
-ELF(.type   _gcry_sm4_aesni_avx2_cfb_dec, at function;)
-_gcry_sm4_aesni_avx2_cfb_dec:
-	/* input:
-	 *	%rdi: ctx, CTX
-	 *	%rsi: dst (16 blocks)
-	 *	%rdx: src (16 blocks)
-	 *	%rcx: iv
-	 */
-	CFI_STARTPROC();
-
-	/* Load input */
-	vmovdqu (%rcx), RNOTx;
-	vinserti128 $1, (%rdx), RNOT, RA0;
-	vmovdqu (0 * 32 + 16)(%rdx), RA1;
-	vmovdqu (1 * 32 + 16)(%rdx), RA2;
-	vmovdqu (2 * 32 + 16)(%rdx), RA3;
-	vmovdqu (3 * 32 + 16)(%rdx), RB0;
-	vmovdqu (4 * 32 + 16)(%rdx), RB1;
-	vmovdqu (5 * 32 + 16)(%rdx), RB2;
-	vmovdqu (6 * 32 + 16)(%rdx), RB3;
-
-	/* Update IV */
-	vmovdqu (7 * 32 + 16)(%rdx), RNOTx;
-	vmovdqu RNOTx, (%rcx);
-
-	call __sm4_crypt_blk16;
-
-	vpxor (0 * 32)(%rdx), RA0, RA0;
-	vpxor (1 * 32)(%rdx), RA1, RA1;
-	vpxor (2 * 32)(%rdx), RA2, RA2;
-	vpxor (3 * 32)(%rdx), RA3, RA3;
-	vpxor (4 * 32)(%rdx), RB0, RB0;
-	vpxor (5 * 32)(%rdx), RB1, RB1;
-	vpxor (6 * 32)(%rdx), RB2, RB2;
-	vpxor (7 * 32)(%rdx), RB3, RB3;
-
-	vmovdqu RA0, (0 * 32)(%rsi);
-	vmovdqu RA1, (1 * 32)(%rsi);
-	vmovdqu RA2, (2 * 32)(%rsi);
-	vmovdqu RA3, (3 * 32)(%rsi);
-	vmovdqu RB0, (4 * 32)(%rsi);
-	vmovdqu RB1, (5 * 32)(%rsi);
-	vmovdqu RB2, (6 * 32)(%rsi);
-	vmovdqu RB3, (7 * 32)(%rsi);
-
-	vzeroall;
-
-	ret_spec_stop;
-	CFI_ENDPROC();
-ELF(.size _gcry_sm4_aesni_avx2_cfb_dec,.-_gcry_sm4_aesni_avx2_cfb_dec;)
-
-.align 16
-.globl _gcry_sm4_aesni_avx2_ocb_enc
-ELF(.type _gcry_sm4_aesni_avx2_ocb_enc, at function;)
-
-_gcry_sm4_aesni_avx2_ocb_enc:
-	/* input:
-	 *	%rdi: ctx, CTX
-	 *	%rsi: dst (16 blocks)
-	 *	%rdx: src (16 blocks)
-	 *	%rcx: offset
-	 *	%r8 : checksum
-	 *	%r9 : L pointers (void *L[16])
-	 */
-	CFI_STARTPROC();
-
-	subq $(4 * 8), %rsp;
-	CFI_ADJUST_CFA_OFFSET(4 * 8);
-
-	movq %r10, (0 * 8)(%rsp);
-	movq %r11, (1 * 8)(%rsp);
-	movq %r12, (2 * 8)(%rsp);
-	movq %r13, (3 * 8)(%rsp);
-	CFI_REL_OFFSET(%r10, 0 * 8);
-	CFI_REL_OFFSET(%r11, 1 * 8);
-	CFI_REL_OFFSET(%r12, 2 * 8);
-	CFI_REL_OFFSET(%r13, 3 * 8);
-
-	vmovdqu (%rcx), RTMP0x;
-	vmovdqu (%r8), RTMP1x;
-
-	/* Offset_i = Offset_{i-1} xor L_{ntz(i)} */
-	/* Checksum_i = Checksum_{i-1} xor P_i  */
-	/* C_i = Offset_i xor ENCIPHER(K, P_i xor Offset_i)  */
-
-#define OCB_INPUT(n, l0reg, l1reg, yreg) \
-	  vmovdqu (n * 32)(%rdx), yreg; \
-	  vpxor (l0reg), RTMP0x, RNOTx; \
-	  vpxor (l1reg), RNOTx, RTMP0x; \
-	  vinserti128 $1, RTMP0x, RNOT, RNOT; \
-	  vpxor yreg, RTMP1, RTMP1; \
-	  vpxor yreg, RNOT, yreg; \
-	  vmovdqu RNOT, (n * 32)(%rsi);
-
-	movq (0 * 8)(%r9), %r10;
-	movq (1 * 8)(%r9), %r11;
-	movq (2 * 8)(%r9), %r12;
-	movq (3 * 8)(%r9), %r13;
-	OCB_INPUT(0, %r10, %r11, RA0);
-	OCB_INPUT(1, %r12, %r13, RA1);
-	movq (4 * 8)(%r9), %r10;
-	movq (5 * 8)(%r9), %r11;
-	movq (6 * 8)(%r9), %r12;
-	movq (7 * 8)(%r9), %r13;
-	OCB_INPUT(2, %r10, %r11, RA2);
-	OCB_INPUT(3, %r12, %r13, RA3);
-	movq (8 * 8)(%r9), %r10;
-	movq (9 * 8)(%r9), %r11;
-	movq (10 * 8)(%r9), %r12;
-	movq (11 * 8)(%r9), %r13;
-	OCB_INPUT(4, %r10, %r11, RB0);
-	OCB_INPUT(5, %r12, %r13, RB1);
-	movq (12 * 8)(%r9), %r10;
-	movq (13 * 8)(%r9), %r11;
-	movq (14 * 8)(%r9), %r12;
-	movq (15 * 8)(%r9), %r13;
-	OCB_INPUT(6, %r10, %r11, RB2);
-	OCB_INPUT(7, %r12, %r13, RB3);
-#undef OCB_INPUT
-
-	vextracti128 $1, RTMP1, RNOTx;
-	vmovdqu RTMP0x, (%rcx);
-	vpxor RNOTx, RTMP1x, RTMP1x;
-	vmovdqu RTMP1x, (%r8);
-
-	movq (0 * 8)(%rsp), %r10;
-	movq (1 * 8)(%rsp), %r11;
-	movq (2 * 8)(%rsp), %r12;
-	movq (3 * 8)(%rsp), %r13;
-	CFI_RESTORE(%r10);
-	CFI_RESTORE(%r11);
-	CFI_RESTORE(%r12);
-	CFI_RESTORE(%r13);
-
-	call __sm4_crypt_blk16;
-
-	addq $(4 * 8), %rsp;
-	CFI_ADJUST_CFA_OFFSET(-4 * 8);
-
-	vpxor (0 * 32)(%rsi), RA0, RA0;
-	vpxor (1 * 32)(%rsi), RA1, RA1;
-	vpxor (2 * 32)(%rsi), RA2, RA2;
-	vpxor (3 * 32)(%rsi), RA3, RA3;
-	vpxor (4 * 32)(%rsi), RB0, RB0;
-	vpxor (5 * 32)(%rsi), RB1, RB1;
-	vpxor (6 * 32)(%rsi), RB2, RB2;
-	vpxor (7 * 32)(%rsi), RB3, RB3;
-
-	vmovdqu RA0, (0 * 32)(%rsi);
-	vmovdqu RA1, (1 * 32)(%rsi);
-	vmovdqu RA2, (2 * 32)(%rsi);
-	vmovdqu RA3, (3 * 32)(%rsi);
-	vmovdqu RB0, (4 * 32)(%rsi);
-	vmovdqu RB1, (5 * 32)(%rsi);
-	vmovdqu RB2, (6 * 32)(%rsi);
-	vmovdqu RB3, (7 * 32)(%rsi);
-
-	vzeroall;
-
-	ret_spec_stop;
-	CFI_ENDPROC();
-ELF(.size _gcry_sm4_aesni_avx2_ocb_enc,.-_gcry_sm4_aesni_avx2_ocb_enc;)
-
-.align 16
-.globl _gcry_sm4_aesni_avx2_ocb_dec
-ELF(.type _gcry_sm4_aesni_avx2_ocb_dec, at function;)
-
-_gcry_sm4_aesni_avx2_ocb_dec:
-	/* input:
-	 *	%rdi: ctx, CTX
-	 *	%rsi: dst (16 blocks)
-	 *	%rdx: src (16 blocks)
-	 *	%rcx: offset
-	 *	%r8 : checksum
-	 *	%r9 : L pointers (void *L[16])
-	 */
-	CFI_STARTPROC();
-
-	subq $(4 * 8), %rsp;
-	CFI_ADJUST_CFA_OFFSET(4 * 8);
-
-	movq %r10, (0 * 8)(%rsp);
-	movq %r11, (1 * 8)(%rsp);
-	movq %r12, (2 * 8)(%rsp);
-	movq %r13, (3 * 8)(%rsp);
-	CFI_REL_OFFSET(%r10, 0 * 8);
-	CFI_REL_OFFSET(%r11, 1 * 8);
-	CFI_REL_OFFSET(%r12, 2 * 8);
-	CFI_REL_OFFSET(%r13, 3 * 8);
-
-	vmovdqu (%rcx), RTMP0x;
-
-	/* Offset_i = Offset_{i-1} xor L_{ntz(i)} */
-	/* C_i = Offset_i xor ENCIPHER(K, P_i xor Offset_i)  */
-
-#define OCB_INPUT(n, l0reg, l1reg, yreg) \
-	  vmovdqu (n * 32)(%rdx), yreg; \
-	  vpxor (l0reg), RTMP0x, RNOTx; \
-	  vpxor (l1reg), RNOTx, RTMP0x; \
-	  vinserti128 $1, RTMP0x, RNOT, RNOT; \
-	  vpxor yreg, RNOT, yreg; \
-	  vmovdqu RNOT, (n * 32)(%rsi);
-
-	movq (0 * 8)(%r9), %r10;
-	movq (1 * 8)(%r9), %r11;
-	movq (2 * 8)(%r9), %r12;
-	movq (3 * 8)(%r9), %r13;
-	OCB_INPUT(0, %r10, %r11, RA0);
-	OCB_INPUT(1, %r12, %r13, RA1);
-	movq (4 * 8)(%r9), %r10;
-	movq (5 * 8)(%r9), %r11;
-	movq (6 * 8)(%r9), %r12;
-	movq (7 * 8)(%r9), %r13;
-	OCB_INPUT(2, %r10, %r11, RA2);
-	OCB_INPUT(3, %r12, %r13, RA3);
-	movq (8 * 8)(%r9), %r10;
-	movq (9 * 8)(%r9), %r11;
-	movq (10 * 8)(%r9), %r12;
-	movq (11 * 8)(%r9), %r13;
-	OCB_INPUT(4, %r10, %r11, RB0);
-	OCB_INPUT(5, %r12, %r13, RB1);
-	movq (12 * 8)(%r9), %r10;
-	movq (13 * 8)(%r9), %r11;
-	movq (14 * 8)(%r9), %r12;
-	movq (15 * 8)(%r9), %r13;
-	OCB_INPUT(6, %r10, %r11, RB2);
-	OCB_INPUT(7, %r12, %r13, RB3);
-#undef OCB_INPUT
-
-	vmovdqu RTMP0x, (%rcx);
-
-	movq (0 * 8)(%rsp), %r10;
-	movq (1 * 8)(%rsp), %r11;
-	movq (2 * 8)(%rsp), %r12;
-	movq (3 * 8)(%rsp), %r13;
-	CFI_RESTORE(%r10);
-	CFI_RESTORE(%r11);
-	CFI_RESTORE(%r12);
-	CFI_RESTORE(%r13);
-
-	call __sm4_crypt_blk16;
-
-	addq $(4 * 8), %rsp;
-	CFI_ADJUST_CFA_OFFSET(-4 * 8);
-
-	vmovdqu (%r8), RTMP1x;
-
-	vpxor (0 * 32)(%rsi), RA0, RA0;
-	vpxor (1 * 32)(%rsi), RA1, RA1;
-	vpxor (2 * 32)(%rsi), RA2, RA2;
-	vpxor (3 * 32)(%rsi), RA3, RA3;
-	vpxor (4 * 32)(%rsi), RB0, RB0;
-	vpxor (5 * 32)(%rsi), RB1, RB1;
-	vpxor (6 * 32)(%rsi), RB2, RB2;
-	vpxor (7 * 32)(%rsi), RB3, RB3;
-
-	/* Checksum_i = Checksum_{i-1} xor P_i  */
-
-	vmovdqu RA0, (0 * 32)(%rsi);
-	vpxor RA0, RTMP1, RTMP1;
-	vmovdqu RA1, (1 * 32)(%rsi);
-	vpxor RA1, RTMP1, RTMP1;
-	vmovdqu RA2, (2 * 32)(%rsi);
-	vpxor RA2, RTMP1, RTMP1;
-	vmovdqu RA3, (3 * 32)(%rsi);
-	vpxor RA3, RTMP1, RTMP1;
-	vmovdqu RB0, (4 * 32)(%rsi);
-	vpxor RB0, RTMP1, RTMP1;
-	vmovdqu RB1, (5 * 32)(%rsi);
-	vpxor RB1, RTMP1, RTMP1;
-	vmovdqu RB2, (6 * 32)(%rsi);
-	vpxor RB2, RTMP1, RTMP1;
-	vmovdqu RB3, (7 * 32)(%rsi);
-	vpxor RB3, RTMP1, RTMP1;
-
-	vextracti128 $1, RTMP1, RNOTx;
-	vpxor RNOTx, RTMP1x, RTMP1x;
-	vmovdqu RTMP1x, (%r8);
-
-	vzeroall;
-
-	ret_spec_stop;
-	CFI_ENDPROC();
-ELF(.size _gcry_sm4_aesni_avx2_ocb_dec,.-_gcry_sm4_aesni_avx2_ocb_dec;)
-
-.align 16
-.globl _gcry_sm4_aesni_avx2_ocb_auth
-ELF(.type _gcry_sm4_aesni_avx2_ocb_auth, at function;)
-
-_gcry_sm4_aesni_avx2_ocb_auth:
-	/* input:
-	 *	%rdi: ctx, CTX
-	 *	%rsi: abuf (16 blocks)
-	 *	%rdx: offset
-	 *	%rcx: checksum
-	 *	%r8 : L pointers (void *L[16])
-	 */
-	CFI_STARTPROC();
-
-	subq $(4 * 8), %rsp;
-	CFI_ADJUST_CFA_OFFSET(4 * 8);
-
-	movq %r10, (0 * 8)(%rsp);
-	movq %r11, (1 * 8)(%rsp);
-	movq %r12, (2 * 8)(%rsp);
-	movq %r13, (3 * 8)(%rsp);
-	CFI_REL_OFFSET(%r10, 0 * 8);
-	CFI_REL_OFFSET(%r11, 1 * 8);
-	CFI_REL_OFFSET(%r12, 2 * 8);
-	CFI_REL_OFFSET(%r13, 3 * 8);
-
-	vmovdqu (%rdx), RTMP0x;
-
-	/* Offset_i = Offset_{i-1} xor L_{ntz(i)} */
-	/* Sum_i = Sum_{i-1} xor ENCIPHER(K, A_i xor Offset_i)  */
-
-#define OCB_INPUT(n, l0reg, l1reg, yreg) \
-	  vmovdqu (n * 32)(%rsi), yreg; \
-	  vpxor (l0reg), RTMP0x, RNOTx; \
-	  vpxor (l1reg), RNOTx, RTMP0x; \
-	  vinserti128 $1, RTMP0x, RNOT, RNOT; \
-	  vpxor yreg, RNOT, yreg;
-
-	movq (0 * 8)(%r8), %r10;
-	movq (1 * 8)(%r8), %r11;
-	movq (2 * 8)(%r8), %r12;
-	movq (3 * 8)(%r8), %r13;
-	OCB_INPUT(0, %r10, %r11, RA0);
-	OCB_INPUT(1, %r12, %r13, RA1);
-	movq (4 * 8)(%r8), %r10;
-	movq (5 * 8)(%r8), %r11;
-	movq (6 * 8)(%r8), %r12;
-	movq (7 * 8)(%r8), %r13;
-	OCB_INPUT(2, %r10, %r11, RA2);
-	OCB_INPUT(3, %r12, %r13, RA3);
-	movq (8 * 8)(%r8), %r10;
-	movq (9 * 8)(%r8), %r11;
-	movq (10 * 8)(%r8), %r12;
-	movq (11 * 8)(%r8), %r13;
-	OCB_INPUT(4, %r10, %r11, RB0);
-	OCB_INPUT(5, %r12, %r13, RB1);
-	movq (12 * 8)(%r8), %r10;
-	movq (13 * 8)(%r8), %r11;
-	movq (14 * 8)(%r8), %r12;
-	movq (15 * 8)(%r8), %r13;
-	OCB_INPUT(6, %r10, %r11, RB2);
-	OCB_INPUT(7, %r12, %r13, RB3);
-#undef OCB_INPUT
-
-	vmovdqu RTMP0x, (%rdx);
-
-	movq (0 * 8)(%rsp), %r10;
-	movq (1 * 8)(%rsp), %r11;
-	movq (2 * 8)(%rsp), %r12;
-	movq (3 * 8)(%rsp), %r13;
-	CFI_RESTORE(%r10);
-	CFI_RESTORE(%r11);
-	CFI_RESTORE(%r12);
-	CFI_RESTORE(%r13);
-
-	call __sm4_crypt_blk16;
-
-	addq $(4 * 8), %rsp;
-	CFI_ADJUST_CFA_OFFSET(-4 * 8);
-
-	vpxor RA0, RB0, RA0;
-	vpxor RA1, RB1, RA1;
-	vpxor RA2, RB2, RA2;
-	vpxor RA3, RB3, RA3;
-
-	vpxor RA1, RA0, RA0;
-	vpxor RA3, RA2, RA2;
-
-	vpxor RA2, RA0, RTMP1;
-
-	vextracti128 $1, RTMP1, RNOTx;
-	vpxor (%rcx), RTMP1x, RTMP1x;
-	vpxor RNOTx, RTMP1x, RTMP1x;
-	vmovdqu RTMP1x, (%rcx);
-
-	vzeroall;
-
-	ret_spec_stop;
-	CFI_ENDPROC();
-ELF(.size _gcry_sm4_aesni_avx2_ocb_auth,.-_gcry_sm4_aesni_avx2_ocb_auth;)
+#define FUNC_NAME(func) _gcry_sm4_aesni_avx2_ ## func
+#define SM4_CRYPT_BLK16 __sm4_aesni_crypt_blk16
+#include "sm4-avx2-amd64.h"
 
 #endif /*defined(ENABLE_AESNI_SUPPORT) && defined(ENABLE_AVX_SUPPORT)*/
 #endif /*__x86_64*/
diff --git a/cipher/sm4-avx2-amd64.h b/cipher/sm4-avx2-amd64.h
new file mode 100644
index 00000000..6d3740dd
--- /dev/null
+++ b/cipher/sm4-avx2-amd64.h
@@ -0,0 +1,648 @@
+/* sm4-avx2-amd64.h  -  Shared AVX2 cipher mode code for SM4 cipher
+ *
+ * Copyright (C) 2020, 2022-2023 Jussi Kivilinna <jussi.kivilinna at iki.fi>
+ *
+ * This file is part of Libgcrypt.
+ *
+ * Libgcrypt is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * Libgcrypt is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef GCRY_SM4_AVX2_AMD64_H
+#define GCRY_SM4_AVX2_AMD64_H
+
+SECTION_RODATA
+.align 32
+
+ELF(.type FUNC_NAME(cipher_mode_consts), at object)
+FUNC_NAME(cipher_mode_consts):
+
+/* CTR byte addition constants */
+.align 32
+.Lbige_addb_0_1:
+	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1
+.Lbige_addb_2_3:
+	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2
+	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3
+.Lbige_addb_4_5:
+	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4
+	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5
+.Lbige_addb_6_7:
+	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6
+	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7
+.Lbige_addb_8_9:
+	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8
+	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9
+.Lbige_addb_10_11:
+	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10
+	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11
+.Lbige_addb_12_13:
+	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12
+	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13
+.Lbige_addb_14_15:
+	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14
+	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15
+
+.text
+
+#define inc_le128(x, minus_one, tmp) \
+	vpcmpeqq minus_one, x, tmp; \
+	vpsubq minus_one, x, x; \
+	vpslldq $8, tmp, tmp; \
+	vpsubq tmp, x, x;
+
+.align 16
+.globl FUNC_NAME(ctr_enc)
+ELF(.type   FUNC_NAME(ctr_enc), at function;)
+FUNC_NAME(ctr_enc):
+	/* input:
+	 *	%rdi: ctx, CTX
+	 *	%rsi: dst (16 blocks)
+	 *	%rdx: src (16 blocks)
+	 *	%rcx: iv (big endian, 128bit)
+	 */
+	CFI_STARTPROC();
+
+	cmpb $(0x100 - 16), 15(%rcx);
+	jbe .Lctr_byteadd;
+
+	movq 8(%rcx), %rax;
+	bswapq %rax;
+
+	vbroadcasti128 .Lbswap128_mask rRIP, RTMP3;
+	vpcmpeqd RNOT, RNOT, RNOT;
+	vpsrldq $8, RNOT, RNOT;   /* ab: -1:0 ; cd: -1:0 */
+	vpaddq RNOT, RNOT, RTMP2; /* ab: -2:0 ; cd: -2:0 */
+
+	/* load IV and byteswap */
+	vmovdqu (%rcx), RTMP4x;
+	vpshufb RTMP3x, RTMP4x, RTMP4x;
+	vmovdqa RTMP4x, RTMP0x;
+	inc_le128(RTMP4x, RNOTx, RTMP1x);
+	vinserti128 $1, RTMP4x, RTMP0, RTMP0;
+	vpshufb RTMP3, RTMP0, RA0; /* +1 ; +0 */
+
+	/* check need for handling 64-bit overflow and carry */
+	cmpq $(0xffffffffffffffff - 16), %rax;
+	ja .Lhandle_ctr_carry;
+
+	/* construct IVs */
+	vpsubq RTMP2, RTMP0, RTMP0; /* +3 ; +2 */
+	vpshufb RTMP3, RTMP0, RA1;
+	vpsubq RTMP2, RTMP0, RTMP0; /* +5 ; +4 */
+	vpshufb RTMP3, RTMP0, RA2;
+	vpsubq RTMP2, RTMP0, RTMP0; /* +7 ; +6 */
+	vpshufb RTMP3, RTMP0, RA3;
+	vpsubq RTMP2, RTMP0, RTMP0; /* +9 ; +8 */
+	vpshufb RTMP3, RTMP0, RB0;
+	vpsubq RTMP2, RTMP0, RTMP0; /* +11 ; +10 */
+	vpshufb RTMP3, RTMP0, RB1;
+	vpsubq RTMP2, RTMP0, RTMP0; /* +13 ; +12 */
+	vpshufb RTMP3, RTMP0, RB2;
+	vpsubq RTMP2, RTMP0, RTMP0; /* +15 ; +14 */
+	vpshufb RTMP3, RTMP0, RB3;
+	vpsubq RTMP2, RTMP0, RTMP0; /* +16 */
+	vpshufb RTMP3x, RTMP0x, RTMP0x;
+
+	jmp .Lctr_carry_done;
+
+.Lhandle_ctr_carry:
+	/* construct IVs */
+	inc_le128(RTMP0, RNOT, RTMP1);
+	inc_le128(RTMP0, RNOT, RTMP1);
+	vpshufb RTMP3, RTMP0, RA1; /* +3 ; +2 */
+	inc_le128(RTMP0, RNOT, RTMP1);
+	inc_le128(RTMP0, RNOT, RTMP1);
+	vpshufb RTMP3, RTMP0, RA2; /* +5 ; +4 */
+	inc_le128(RTMP0, RNOT, RTMP1);
+	inc_le128(RTMP0, RNOT, RTMP1);
+	vpshufb RTMP3, RTMP0, RA3; /* +7 ; +6 */
+	inc_le128(RTMP0, RNOT, RTMP1);
+	inc_le128(RTMP0, RNOT, RTMP1);
+	vpshufb RTMP3, RTMP0, RB0; /* +9 ; +8 */
+	inc_le128(RTMP0, RNOT, RTMP1);
+	inc_le128(RTMP0, RNOT, RTMP1);
+	vpshufb RTMP3, RTMP0, RB1; /* +11 ; +10 */
+	inc_le128(RTMP0, RNOT, RTMP1);
+	inc_le128(RTMP0, RNOT, RTMP1);
+	vpshufb RTMP3, RTMP0, RB2; /* +13 ; +12 */
+	inc_le128(RTMP0, RNOT, RTMP1);
+	inc_le128(RTMP0, RNOT, RTMP1);
+	vpshufb RTMP3, RTMP0, RB3; /* +15 ; +14 */
+	inc_le128(RTMP0, RNOT, RTMP1);
+	vextracti128 $1, RTMP0, RTMP0x;
+	vpshufb RTMP3x, RTMP0x, RTMP0x; /* +16 */
+
+.Lctr_carry_done:
+	/* store new IV */
+	vmovdqu RTMP0x, (%rcx);
+
+.align 8
+.Lload_ctr_done:
+	call SM4_CRYPT_BLK16;
+
+	vpxor (0 * 32)(%rdx), RA0, RA0;
+	vpxor (1 * 32)(%rdx), RA1, RA1;
+	vpxor (2 * 32)(%rdx), RA2, RA2;
+	vpxor (3 * 32)(%rdx), RA3, RA3;
+	vpxor (4 * 32)(%rdx), RB0, RB0;
+	vpxor (5 * 32)(%rdx), RB1, RB1;
+	vpxor (6 * 32)(%rdx), RB2, RB2;
+	vpxor (7 * 32)(%rdx), RB3, RB3;
+
+	vmovdqu RA0, (0 * 32)(%rsi);
+	vmovdqu RA1, (1 * 32)(%rsi);
+	vmovdqu RA2, (2 * 32)(%rsi);
+	vmovdqu RA3, (3 * 32)(%rsi);
+	vmovdqu RB0, (4 * 32)(%rsi);
+	vmovdqu RB1, (5 * 32)(%rsi);
+	vmovdqu RB2, (6 * 32)(%rsi);
+	vmovdqu RB3, (7 * 32)(%rsi);
+
+	vzeroall;
+
+	ret_spec_stop;
+
+.align 8
+.Lctr_byteadd_full_ctr_carry:
+	movq 8(%rcx), %r11;
+	movq (%rcx), %r10;
+	bswapq %r11;
+	bswapq %r10;
+	addq $16, %r11;
+	adcq $0, %r10;
+	bswapq %r11;
+	bswapq %r10;
+	movq %r11, 8(%rcx);
+	movq %r10, (%rcx);
+	jmp .Lctr_byteadd_ymm;
+.align 8
+.Lctr_byteadd:
+	vbroadcasti128 (%rcx), RB3;
+	je .Lctr_byteadd_full_ctr_carry;
+	addb $16, 15(%rcx);
+.Lctr_byteadd_ymm:
+	vpaddb .Lbige_addb_0_1 rRIP, RB3, RA0;
+	vpaddb .Lbige_addb_2_3 rRIP, RB3, RA1;
+	vpaddb .Lbige_addb_4_5 rRIP, RB3, RA2;
+	vpaddb .Lbige_addb_6_7 rRIP, RB3, RA3;
+	vpaddb .Lbige_addb_8_9 rRIP, RB3, RB0;
+	vpaddb .Lbige_addb_10_11 rRIP, RB3, RB1;
+	vpaddb .Lbige_addb_12_13 rRIP, RB3, RB2;
+	vpaddb .Lbige_addb_14_15 rRIP, RB3, RB3;
+
+	jmp .Lload_ctr_done;
+	CFI_ENDPROC();
+ELF(.size FUNC_NAME(ctr_enc),.-FUNC_NAME(ctr_enc);)
+
+.align 16
+.globl FUNC_NAME(cbc_dec)
+ELF(.type   FUNC_NAME(cbc_dec), at function;)
+FUNC_NAME(cbc_dec):
+	/* input:
+	 *	%rdi: ctx, CTX
+	 *	%rsi: dst (16 blocks)
+	 *	%rdx: src (16 blocks)
+	 *	%rcx: iv
+	 */
+	CFI_STARTPROC();
+
+	vmovdqu (0 * 32)(%rdx), RA0;
+	vmovdqu (1 * 32)(%rdx), RA1;
+	vmovdqu (2 * 32)(%rdx), RA2;
+	vmovdqu (3 * 32)(%rdx), RA3;
+	vmovdqu (4 * 32)(%rdx), RB0;
+	vmovdqu (5 * 32)(%rdx), RB1;
+	vmovdqu (6 * 32)(%rdx), RB2;
+	vmovdqu (7 * 32)(%rdx), RB3;
+
+	call SM4_CRYPT_BLK16;
+
+	vmovdqu (%rcx), RNOTx;
+	vinserti128 $1, (%rdx), RNOT, RNOT;
+	vpxor RNOT, RA0, RA0;
+	vpxor (0 * 32 + 16)(%rdx), RA1, RA1;
+	vpxor (1 * 32 + 16)(%rdx), RA2, RA2;
+	vpxor (2 * 32 + 16)(%rdx), RA3, RA3;
+	vpxor (3 * 32 + 16)(%rdx), RB0, RB0;
+	vpxor (4 * 32 + 16)(%rdx), RB1, RB1;
+	vpxor (5 * 32 + 16)(%rdx), RB2, RB2;
+	vpxor (6 * 32 + 16)(%rdx), RB3, RB3;
+	vmovdqu (7 * 32 + 16)(%rdx), RNOTx;
+	vmovdqu RNOTx, (%rcx); /* store new IV */
+
+	vmovdqu RA0, (0 * 32)(%rsi);
+	vmovdqu RA1, (1 * 32)(%rsi);
+	vmovdqu RA2, (2 * 32)(%rsi);
+	vmovdqu RA3, (3 * 32)(%rsi);
+	vmovdqu RB0, (4 * 32)(%rsi);
+	vmovdqu RB1, (5 * 32)(%rsi);
+	vmovdqu RB2, (6 * 32)(%rsi);
+	vmovdqu RB3, (7 * 32)(%rsi);
+
+	vzeroall;
+
+	ret_spec_stop;
+	CFI_ENDPROC();
+ELF(.size FUNC_NAME(cbc_dec),.-FUNC_NAME(cbc_dec);)
+
+.align 16
+.globl FUNC_NAME(cfb_dec)
+ELF(.type   FUNC_NAME(cfb_dec), at function;)
+FUNC_NAME(cfb_dec):
+	/* input:
+	 *	%rdi: ctx, CTX
+	 *	%rsi: dst (16 blocks)
+	 *	%rdx: src (16 blocks)
+	 *	%rcx: iv
+	 */
+	CFI_STARTPROC();
+
+	/* Load input */
+	vmovdqu (%rcx), RNOTx;
+	vinserti128 $1, (%rdx), RNOT, RA0;
+	vmovdqu (0 * 32 + 16)(%rdx), RA1;
+	vmovdqu (1 * 32 + 16)(%rdx), RA2;
+	vmovdqu (2 * 32 + 16)(%rdx), RA3;
+	vmovdqu (3 * 32 + 16)(%rdx), RB0;
+	vmovdqu (4 * 32 + 16)(%rdx), RB1;
+	vmovdqu (5 * 32 + 16)(%rdx), RB2;
+	vmovdqu (6 * 32 + 16)(%rdx), RB3;
+
+	/* Update IV */
+	vmovdqu (7 * 32 + 16)(%rdx), RNOTx;
+	vmovdqu RNOTx, (%rcx);
+
+	call SM4_CRYPT_BLK16;
+
+	vpxor (0 * 32)(%rdx), RA0, RA0;
+	vpxor (1 * 32)(%rdx), RA1, RA1;
+	vpxor (2 * 32)(%rdx), RA2, RA2;
+	vpxor (3 * 32)(%rdx), RA3, RA3;
+	vpxor (4 * 32)(%rdx), RB0, RB0;
+	vpxor (5 * 32)(%rdx), RB1, RB1;
+	vpxor (6 * 32)(%rdx), RB2, RB2;
+	vpxor (7 * 32)(%rdx), RB3, RB3;
+
+	vmovdqu RA0, (0 * 32)(%rsi);
+	vmovdqu RA1, (1 * 32)(%rsi);
+	vmovdqu RA2, (2 * 32)(%rsi);
+	vmovdqu RA3, (3 * 32)(%rsi);
+	vmovdqu RB0, (4 * 32)(%rsi);
+	vmovdqu RB1, (5 * 32)(%rsi);
+	vmovdqu RB2, (6 * 32)(%rsi);
+	vmovdqu RB3, (7 * 32)(%rsi);
+
+	vzeroall;
+
+	ret_spec_stop;
+	CFI_ENDPROC();
+ELF(.size FUNC_NAME(cfb_dec),.-FUNC_NAME(cfb_dec);)
+
+.align 16
+.globl FUNC_NAME(ocb_enc)
+ELF(.type FUNC_NAME(ocb_enc), at function;)
+
+FUNC_NAME(ocb_enc):
+	/* input:
+	 *	%rdi: ctx, CTX
+	 *	%rsi: dst (16 blocks)
+	 *	%rdx: src (16 blocks)
+	 *	%rcx: offset
+	 *	%r8 : checksum
+	 *	%r9 : L pointers (void *L[16])
+	 */
+	CFI_STARTPROC();
+
+	subq $(4 * 8), %rsp;
+	CFI_ADJUST_CFA_OFFSET(4 * 8);
+
+	movq %r10, (0 * 8)(%rsp);
+	movq %r11, (1 * 8)(%rsp);
+	movq %r12, (2 * 8)(%rsp);
+	movq %r13, (3 * 8)(%rsp);
+	CFI_REL_OFFSET(%r10, 0 * 8);
+	CFI_REL_OFFSET(%r11, 1 * 8);
+	CFI_REL_OFFSET(%r12, 2 * 8);
+	CFI_REL_OFFSET(%r13, 3 * 8);
+
+	vmovdqu (%rcx), RTMP0x;
+	vmovdqu (%r8), RTMP1x;
+
+	/* Offset_i = Offset_{i-1} xor L_{ntz(i)} */
+	/* Checksum_i = Checksum_{i-1} xor P_i  */
+	/* C_i = Offset_i xor ENCIPHER(K, P_i xor Offset_i)  */
+
+#define OCB_INPUT(n, l0reg, l1reg, yreg) \
+	  vmovdqu (n * 32)(%rdx), yreg; \
+	  vpxor (l0reg), RTMP0x, RNOTx; \
+	  vpxor (l1reg), RNOTx, RTMP0x; \
+	  vinserti128 $1, RTMP0x, RNOT, RNOT; \
+	  vpxor yreg, RTMP1, RTMP1; \
+	  vpxor yreg, RNOT, yreg; \
+	  vmovdqu RNOT, (n * 32)(%rsi);
+
+	movq (0 * 8)(%r9), %r10;
+	movq (1 * 8)(%r9), %r11;
+	movq (2 * 8)(%r9), %r12;
+	movq (3 * 8)(%r9), %r13;
+	OCB_INPUT(0, %r10, %r11, RA0);
+	OCB_INPUT(1, %r12, %r13, RA1);
+	movq (4 * 8)(%r9), %r10;
+	movq (5 * 8)(%r9), %r11;
+	movq (6 * 8)(%r9), %r12;
+	movq (7 * 8)(%r9), %r13;
+	OCB_INPUT(2, %r10, %r11, RA2);
+	OCB_INPUT(3, %r12, %r13, RA3);
+	movq (8 * 8)(%r9), %r10;
+	movq (9 * 8)(%r9), %r11;
+	movq (10 * 8)(%r9), %r12;
+	movq (11 * 8)(%r9), %r13;
+	OCB_INPUT(4, %r10, %r11, RB0);
+	OCB_INPUT(5, %r12, %r13, RB1);
+	movq (12 * 8)(%r9), %r10;
+	movq (13 * 8)(%r9), %r11;
+	movq (14 * 8)(%r9), %r12;
+	movq (15 * 8)(%r9), %r13;
+	OCB_INPUT(6, %r10, %r11, RB2);
+	OCB_INPUT(7, %r12, %r13, RB3);
+#undef OCB_INPUT
+
+	vextracti128 $1, RTMP1, RNOTx;
+	vmovdqu RTMP0x, (%rcx);
+	vpxor RNOTx, RTMP1x, RTMP1x;
+	vmovdqu RTMP1x, (%r8);
+
+	movq (0 * 8)(%rsp), %r10;
+	movq (1 * 8)(%rsp), %r11;
+	movq (2 * 8)(%rsp), %r12;
+	movq (3 * 8)(%rsp), %r13;
+	CFI_RESTORE(%r10);
+	CFI_RESTORE(%r11);
+	CFI_RESTORE(%r12);
+	CFI_RESTORE(%r13);
+
+	call SM4_CRYPT_BLK16;
+
+	addq $(4 * 8), %rsp;
+	CFI_ADJUST_CFA_OFFSET(-4 * 8);
+
+	vpxor (0 * 32)(%rsi), RA0, RA0;
+	vpxor (1 * 32)(%rsi), RA1, RA1;
+	vpxor (2 * 32)(%rsi), RA2, RA2;
+	vpxor (3 * 32)(%rsi), RA3, RA3;
+	vpxor (4 * 32)(%rsi), RB0, RB0;
+	vpxor (5 * 32)(%rsi), RB1, RB1;
+	vpxor (6 * 32)(%rsi), RB2, RB2;
+	vpxor (7 * 32)(%rsi), RB3, RB3;
+
+	vmovdqu RA0, (0 * 32)(%rsi);
+	vmovdqu RA1, (1 * 32)(%rsi);
+	vmovdqu RA2, (2 * 32)(%rsi);
+	vmovdqu RA3, (3 * 32)(%rsi);
+	vmovdqu RB0, (4 * 32)(%rsi);
+	vmovdqu RB1, (5 * 32)(%rsi);
+	vmovdqu RB2, (6 * 32)(%rsi);
+	vmovdqu RB3, (7 * 32)(%rsi);
+
+	vzeroall;
+
+	ret_spec_stop;
+	CFI_ENDPROC();
+ELF(.size FUNC_NAME(ocb_enc),.-FUNC_NAME(ocb_enc);)
+
+.align 16
+.globl FUNC_NAME(ocb_dec)
+ELF(.type FUNC_NAME(ocb_dec), at function;)
+
+FUNC_NAME(ocb_dec):
+	/* input:
+	 *	%rdi: ctx, CTX
+	 *	%rsi: dst (16 blocks)
+	 *	%rdx: src (16 blocks)
+	 *	%rcx: offset
+	 *	%r8 : checksum
+	 *	%r9 : L pointers (void *L[16])
+	 */
+	CFI_STARTPROC();
+
+	subq $(4 * 8), %rsp;
+	CFI_ADJUST_CFA_OFFSET(4 * 8);
+
+	movq %r10, (0 * 8)(%rsp);
+	movq %r11, (1 * 8)(%rsp);
+	movq %r12, (2 * 8)(%rsp);
+	movq %r13, (3 * 8)(%rsp);
+	CFI_REL_OFFSET(%r10, 0 * 8);
+	CFI_REL_OFFSET(%r11, 1 * 8);
+	CFI_REL_OFFSET(%r12, 2 * 8);
+	CFI_REL_OFFSET(%r13, 3 * 8);
+
+	vmovdqu (%rcx), RTMP0x;
+
+	/* Offset_i = Offset_{i-1} xor L_{ntz(i)} */
+	/* C_i = Offset_i xor ENCIPHER(K, P_i xor Offset_i)  */
+
+#define OCB_INPUT(n, l0reg, l1reg, yreg) \
+	  vmovdqu (n * 32)(%rdx), yreg; \
+	  vpxor (l0reg), RTMP0x, RNOTx; \
+	  vpxor (l1reg), RNOTx, RTMP0x; \
+	  vinserti128 $1, RTMP0x, RNOT, RNOT; \
+	  vpxor yreg, RNOT, yreg; \
+	  vmovdqu RNOT, (n * 32)(%rsi);
+
+	movq (0 * 8)(%r9), %r10;
+	movq (1 * 8)(%r9), %r11;
+	movq (2 * 8)(%r9), %r12;
+	movq (3 * 8)(%r9), %r13;
+	OCB_INPUT(0, %r10, %r11, RA0);
+	OCB_INPUT(1, %r12, %r13, RA1);
+	movq (4 * 8)(%r9), %r10;
+	movq (5 * 8)(%r9), %r11;
+	movq (6 * 8)(%r9), %r12;
+	movq (7 * 8)(%r9), %r13;
+	OCB_INPUT(2, %r10, %r11, RA2);
+	OCB_INPUT(3, %r12, %r13, RA3);
+	movq (8 * 8)(%r9), %r10;
+	movq (9 * 8)(%r9), %r11;
+	movq (10 * 8)(%r9), %r12;
+	movq (11 * 8)(%r9), %r13;
+	OCB_INPUT(4, %r10, %r11, RB0);
+	OCB_INPUT(5, %r12, %r13, RB1);
+	movq (12 * 8)(%r9), %r10;
+	movq (13 * 8)(%r9), %r11;
+	movq (14 * 8)(%r9), %r12;
+	movq (15 * 8)(%r9), %r13;
+	OCB_INPUT(6, %r10, %r11, RB2);
+	OCB_INPUT(7, %r12, %r13, RB3);
+#undef OCB_INPUT
+
+	vmovdqu RTMP0x, (%rcx);
+
+	movq (0 * 8)(%rsp), %r10;
+	movq (1 * 8)(%rsp), %r11;
+	movq (2 * 8)(%rsp), %r12;
+	movq (3 * 8)(%rsp), %r13;
+	CFI_RESTORE(%r10);
+	CFI_RESTORE(%r11);
+	CFI_RESTORE(%r12);
+	CFI_RESTORE(%r13);
+
+	call SM4_CRYPT_BLK16;
+
+	addq $(4 * 8), %rsp;
+	CFI_ADJUST_CFA_OFFSET(-4 * 8);
+
+	vmovdqu (%r8), RTMP1x;
+
+	vpxor (0 * 32)(%rsi), RA0, RA0;
+	vpxor (1 * 32)(%rsi), RA1, RA1;
+	vpxor (2 * 32)(%rsi), RA2, RA2;
+	vpxor (3 * 32)(%rsi), RA3, RA3;
+	vpxor (4 * 32)(%rsi), RB0, RB0;
+	vpxor (5 * 32)(%rsi), RB1, RB1;
+	vpxor (6 * 32)(%rsi), RB2, RB2;
+	vpxor (7 * 32)(%rsi), RB3, RB3;
+
+	/* Checksum_i = Checksum_{i-1} xor P_i  */
+
+	vmovdqu RA0, (0 * 32)(%rsi);
+	vpxor RA0, RTMP1, RTMP1;
+	vmovdqu RA1, (1 * 32)(%rsi);
+	vpxor RA1, RTMP1, RTMP1;
+	vmovdqu RA2, (2 * 32)(%rsi);
+	vpxor RA2, RTMP1, RTMP1;
+	vmovdqu RA3, (3 * 32)(%rsi);
+	vpxor RA3, RTMP1, RTMP1;
+	vmovdqu RB0, (4 * 32)(%rsi);
+	vpxor RB0, RTMP1, RTMP1;
+	vmovdqu RB1, (5 * 32)(%rsi);
+	vpxor RB1, RTMP1, RTMP1;
+	vmovdqu RB2, (6 * 32)(%rsi);
+	vpxor RB2, RTMP1, RTMP1;
+	vmovdqu RB3, (7 * 32)(%rsi);
+	vpxor RB3, RTMP1, RTMP1;
+
+	vextracti128 $1, RTMP1, RNOTx;
+	vpxor RNOTx, RTMP1x, RTMP1x;
+	vmovdqu RTMP1x, (%r8);
+
+	vzeroall;
+
+	ret_spec_stop;
+	CFI_ENDPROC();
+ELF(.size FUNC_NAME(ocb_dec),.-FUNC_NAME(ocb_dec);)
+
+.align 16
+.globl FUNC_NAME(ocb_auth)
+ELF(.type FUNC_NAME(ocb_auth), at function;)
+
+FUNC_NAME(ocb_auth):
+	/* input:
+	 *	%rdi: ctx, CTX
+	 *	%rsi: abuf (16 blocks)
+	 *	%rdx: offset
+	 *	%rcx: checksum
+	 *	%r8 : L pointers (void *L[16])
+	 */
+	CFI_STARTPROC();
+
+	subq $(4 * 8), %rsp;
+	CFI_ADJUST_CFA_OFFSET(4 * 8);
+
+	movq %r10, (0 * 8)(%rsp);
+	movq %r11, (1 * 8)(%rsp);
+	movq %r12, (2 * 8)(%rsp);
+	movq %r13, (3 * 8)(%rsp);
+	CFI_REL_OFFSET(%r10, 0 * 8);
+	CFI_REL_OFFSET(%r11, 1 * 8);
+	CFI_REL_OFFSET(%r12, 2 * 8);
+	CFI_REL_OFFSET(%r13, 3 * 8);
+
+	vmovdqu (%rdx), RTMP0x;
+
+	/* Offset_i = Offset_{i-1} xor L_{ntz(i)} */
+	/* Sum_i = Sum_{i-1} xor ENCIPHER(K, A_i xor Offset_i)  */
+
+#define OCB_INPUT(n, l0reg, l1reg, yreg) \
+	  vmovdqu (n * 32)(%rsi), yreg; \
+	  vpxor (l0reg), RTMP0x, RNOTx; \
+	  vpxor (l1reg), RNOTx, RTMP0x; \
+	  vinserti128 $1, RTMP0x, RNOT, RNOT; \
+	  vpxor yreg, RNOT, yreg;
+
+	movq (0 * 8)(%r8), %r10;
+	movq (1 * 8)(%r8), %r11;
+	movq (2 * 8)(%r8), %r12;
+	movq (3 * 8)(%r8), %r13;
+	OCB_INPUT(0, %r10, %r11, RA0);
+	OCB_INPUT(1, %r12, %r13, RA1);
+	movq (4 * 8)(%r8), %r10;
+	movq (5 * 8)(%r8), %r11;
+	movq (6 * 8)(%r8), %r12;
+	movq (7 * 8)(%r8), %r13;
+	OCB_INPUT(2, %r10, %r11, RA2);
+	OCB_INPUT(3, %r12, %r13, RA3);
+	movq (8 * 8)(%r8), %r10;
+	movq (9 * 8)(%r8), %r11;
+	movq (10 * 8)(%r8), %r12;
+	movq (11 * 8)(%r8), %r13;
+	OCB_INPUT(4, %r10, %r11, RB0);
+	OCB_INPUT(5, %r12, %r13, RB1);
+	movq (12 * 8)(%r8), %r10;
+	movq (13 * 8)(%r8), %r11;
+	movq (14 * 8)(%r8), %r12;
+	movq (15 * 8)(%r8), %r13;
+	OCB_INPUT(6, %r10, %r11, RB2);
+	OCB_INPUT(7, %r12, %r13, RB3);
+#undef OCB_INPUT
+
+	vmovdqu RTMP0x, (%rdx);
+
+	movq (0 * 8)(%rsp), %r10;
+	movq (1 * 8)(%rsp), %r11;
+	movq (2 * 8)(%rsp), %r12;
+	movq (3 * 8)(%rsp), %r13;
+	CFI_RESTORE(%r10);
+	CFI_RESTORE(%r11);
+	CFI_RESTORE(%r12);
+	CFI_RESTORE(%r13);
+
+	call SM4_CRYPT_BLK16;
+
+	addq $(4 * 8), %rsp;
+	CFI_ADJUST_CFA_OFFSET(-4 * 8);
+
+	vpxor RA0, RB0, RA0;
+	vpxor RA1, RB1, RA1;
+	vpxor RA2, RB2, RA2;
+	vpxor RA3, RB3, RA3;
+
+	vpxor RA1, RA0, RA0;
+	vpxor RA3, RA2, RA2;
+
+	vpxor RA2, RA0, RTMP1;
+
+	vextracti128 $1, RTMP1, RNOTx;
+	vpxor (%rcx), RTMP1x, RTMP1x;
+	vpxor RNOTx, RTMP1x, RTMP1x;
+	vmovdqu RTMP1x, (%rcx);
+
+	vzeroall;
+
+	ret_spec_stop;
+	CFI_ENDPROC();
+ELF(.size FUNC_NAME(ocb_auth),.-FUNC_NAME(ocb_auth);)
+
+#endif /* GCRY_SM4_AVX2_AMD64_H */
diff --git a/cipher/sm4-gfni-avx2-amd64.S b/cipher/sm4-gfni-avx2-amd64.S
index 464da399..9aff85c6 100644
--- a/cipher/sm4-gfni-avx2-amd64.S
+++ b/cipher/sm4-gfni-avx2-amd64.S
@@ -128,7 +128,7 @@ _sm4_gfni_avx2_consts:
 	.byte 0x01, 0x02, 0x03, 0x00, 0x05, 0x06, 0x07, 0x04
 	.byte 0x09, 0x0a, 0x0b, 0x08, 0x0d, 0x0e, 0x0f, 0x0c
 
-/* For CTR-mode IV byteswap */
+/* For output word byteswap */
 .Lbswap128_mask:
 	.byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
 
@@ -136,33 +136,6 @@ _sm4_gfni_avx2_consts:
 .Lbswap32_mask:
 	.byte 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12
 
-/* CTR byte addition constants */
-.align 32
-.Lbige_addb_0_1:
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1
-.Lbige_addb_2_3:
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3
-.Lbige_addb_4_5:
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5
-.Lbige_addb_6_7:
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7
-.Lbige_addb_8_9:
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9
-.Lbige_addb_10_11:
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11
-.Lbige_addb_12_13:
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13
-.Lbige_addb_14_15:
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15
-
 .text
 
 .align 16
@@ -667,594 +640,9 @@ _gcry_sm4_gfni_avx2_crypt_blk1_16:
 	CFI_ENDPROC();
 ELF(.size _gcry_sm4_gfni_avx2_crypt_blk1_16,.-_gcry_sm4_gfni_avx2_crypt_blk1_16;)
 
-#define inc_le128(x, minus_one, tmp) \
-	vpcmpeqq minus_one, x, tmp; \
-	vpsubq minus_one, x, x; \
-	vpslldq $8, tmp, tmp; \
-	vpsubq tmp, x, x;
-
-.align 16
-.globl _gcry_sm4_gfni_avx2_ctr_enc
-ELF(.type   _gcry_sm4_gfni_avx2_ctr_enc, at function;)
-_gcry_sm4_gfni_avx2_ctr_enc:
-	/* input:
-	 *	%rdi: ctx, CTX
-	 *	%rsi: dst (16 blocks)
-	 *	%rdx: src (16 blocks)
-	 *	%rcx: iv (big endian, 128bit)
-	 */
-	CFI_STARTPROC();
-
-	cmpb $(0x100 - 16), 15(%rcx);
-	jbe .Lctr_byteadd;
-
-	movq 8(%rcx), %rax;
-	bswapq %rax;
-
-	vbroadcasti128 .Lbswap128_mask rRIP, RTMP3;
-	vpcmpeqd RNOT, RNOT, RNOT;
-	vpsrldq $8, RNOT, RNOT;   /* ab: -1:0 ; cd: -1:0 */
-	vpaddq RNOT, RNOT, RTMP2; /* ab: -2:0 ; cd: -2:0 */
-
-	/* load IV and byteswap */
-	vmovdqu (%rcx), RTMP4x;
-	vpshufb RTMP3x, RTMP4x, RTMP4x;
-	vmovdqa RTMP4x, RTMP0x;
-	inc_le128(RTMP4x, RNOTx, RTMP1x);
-	vinserti128 $1, RTMP4x, RTMP0, RTMP0;
-	vpshufb RTMP3, RTMP0, RA0; /* +1 ; +0 */
-
-	/* check need for handling 64-bit overflow and carry */
-	cmpq $(0xffffffffffffffff - 16), %rax;
-	ja .Lhandle_ctr_carry;
-
-	/* construct IVs */
-	vpsubq RTMP2, RTMP0, RTMP0; /* +3 ; +2 */
-	vpshufb RTMP3, RTMP0, RA1;
-	vpsubq RTMP2, RTMP0, RTMP0; /* +5 ; +4 */
-	vpshufb RTMP3, RTMP0, RA2;
-	vpsubq RTMP2, RTMP0, RTMP0; /* +7 ; +6 */
-	vpshufb RTMP3, RTMP0, RA3;
-	vpsubq RTMP2, RTMP0, RTMP0; /* +9 ; +8 */
-	vpshufb RTMP3, RTMP0, RB0;
-	vpsubq RTMP2, RTMP0, RTMP0; /* +11 ; +10 */
-	vpshufb RTMP3, RTMP0, RB1;
-	vpsubq RTMP2, RTMP0, RTMP0; /* +13 ; +12 */
-	vpshufb RTMP3, RTMP0, RB2;
-	vpsubq RTMP2, RTMP0, RTMP0; /* +15 ; +14 */
-	vpshufb RTMP3, RTMP0, RB3;
-	vpsubq RTMP2, RTMP0, RTMP0; /* +16 */
-	vpshufb RTMP3x, RTMP0x, RTMP0x;
-
-	jmp .Lctr_carry_done;
-
-.Lhandle_ctr_carry:
-	/* construct IVs */
-	inc_le128(RTMP0, RNOT, RTMP1);
-	inc_le128(RTMP0, RNOT, RTMP1);
-	vpshufb RTMP3, RTMP0, RA1; /* +3 ; +2 */
-	inc_le128(RTMP0, RNOT, RTMP1);
-	inc_le128(RTMP0, RNOT, RTMP1);
-	vpshufb RTMP3, RTMP0, RA2; /* +5 ; +4 */
-	inc_le128(RTMP0, RNOT, RTMP1);
-	inc_le128(RTMP0, RNOT, RTMP1);
-	vpshufb RTMP3, RTMP0, RA3; /* +7 ; +6 */
-	inc_le128(RTMP0, RNOT, RTMP1);
-	inc_le128(RTMP0, RNOT, RTMP1);
-	vpshufb RTMP3, RTMP0, RB0; /* +9 ; +8 */
-	inc_le128(RTMP0, RNOT, RTMP1);
-	inc_le128(RTMP0, RNOT, RTMP1);
-	vpshufb RTMP3, RTMP0, RB1; /* +11 ; +10 */
-	inc_le128(RTMP0, RNOT, RTMP1);
-	inc_le128(RTMP0, RNOT, RTMP1);
-	vpshufb RTMP3, RTMP0, RB2; /* +13 ; +12 */
-	inc_le128(RTMP0, RNOT, RTMP1);
-	inc_le128(RTMP0, RNOT, RTMP1);
-	vpshufb RTMP3, RTMP0, RB3; /* +15 ; +14 */
-	inc_le128(RTMP0, RNOT, RTMP1);
-	vextracti128 $1, RTMP0, RTMP0x;
-	vpshufb RTMP3x, RTMP0x, RTMP0x; /* +16 */
-
-.Lctr_carry_done:
-	/* store new IV */
-	vmovdqu RTMP0x, (%rcx);
-
-.align 8
-.Lload_ctr_done:
-	call __sm4_gfni_crypt_blk16;
-
-	vpxor (0 * 32)(%rdx), RA0, RA0;
-	vpxor (1 * 32)(%rdx), RA1, RA1;
-	vpxor (2 * 32)(%rdx), RA2, RA2;
-	vpxor (3 * 32)(%rdx), RA3, RA3;
-	vpxor (4 * 32)(%rdx), RB0, RB0;
-	vpxor (5 * 32)(%rdx), RB1, RB1;
-	vpxor (6 * 32)(%rdx), RB2, RB2;
-	vpxor (7 * 32)(%rdx), RB3, RB3;
-
-	vmovdqu RA0, (0 * 32)(%rsi);
-	vmovdqu RA1, (1 * 32)(%rsi);
-	vmovdqu RA2, (2 * 32)(%rsi);
-	vmovdqu RA3, (3 * 32)(%rsi);
-	vmovdqu RB0, (4 * 32)(%rsi);
-	vmovdqu RB1, (5 * 32)(%rsi);
-	vmovdqu RB2, (6 * 32)(%rsi);
-	vmovdqu RB3, (7 * 32)(%rsi);
-
-	vzeroall;
-
-	ret_spec_stop;
-
-.align 8
-.Lctr_byteadd_full_ctr_carry:
-	movq 8(%rcx), %r11;
-	movq (%rcx), %r10;
-	bswapq %r11;
-	bswapq %r10;
-	addq $16, %r11;
-	adcq $0, %r10;
-	bswapq %r11;
-	bswapq %r10;
-	movq %r11, 8(%rcx);
-	movq %r10, (%rcx);
-	jmp .Lctr_byteadd_ymm;
-.align 8
-.Lctr_byteadd:
-	vbroadcasti128 (%rcx), RB3;
-	je .Lctr_byteadd_full_ctr_carry;
-	addb $16, 15(%rcx);
-.Lctr_byteadd_ymm:
-	vpaddb .Lbige_addb_0_1 rRIP, RB3, RA0;
-	vpaddb .Lbige_addb_2_3 rRIP, RB3, RA1;
-	vpaddb .Lbige_addb_4_5 rRIP, RB3, RA2;
-	vpaddb .Lbige_addb_6_7 rRIP, RB3, RA3;
-	vpaddb .Lbige_addb_8_9 rRIP, RB3, RB0;
-	vpaddb .Lbige_addb_10_11 rRIP, RB3, RB1;
-	vpaddb .Lbige_addb_12_13 rRIP, RB3, RB2;
-	vpaddb .Lbige_addb_14_15 rRIP, RB3, RB3;
-
-	jmp .Lload_ctr_done;
-	CFI_ENDPROC();
-ELF(.size _gcry_sm4_gfni_avx2_ctr_enc,.-_gcry_sm4_gfni_avx2_ctr_enc;)
-
-.align 16
-.globl _gcry_sm4_gfni_avx2_cbc_dec
-ELF(.type   _gcry_sm4_gfni_avx2_cbc_dec, at function;)
-_gcry_sm4_gfni_avx2_cbc_dec:
-	/* input:
-	 *	%rdi: ctx, CTX
-	 *	%rsi: dst (16 blocks)
-	 *	%rdx: src (16 blocks)
-	 *	%rcx: iv
-	 */
-	CFI_STARTPROC();
-
-	vmovdqu (0 * 32)(%rdx), RA0;
-	vmovdqu (1 * 32)(%rdx), RA1;
-	vmovdqu (2 * 32)(%rdx), RA2;
-	vmovdqu (3 * 32)(%rdx), RA3;
-	vmovdqu (4 * 32)(%rdx), RB0;
-	vmovdqu (5 * 32)(%rdx), RB1;
-	vmovdqu (6 * 32)(%rdx), RB2;
-	vmovdqu (7 * 32)(%rdx), RB3;
-
-	call __sm4_gfni_crypt_blk16;
-
-	vmovdqu (%rcx), RNOTx;
-	vinserti128 $1, (%rdx), RNOT, RNOT;
-	vpxor RNOT, RA0, RA0;
-	vpxor (0 * 32 + 16)(%rdx), RA1, RA1;
-	vpxor (1 * 32 + 16)(%rdx), RA2, RA2;
-	vpxor (2 * 32 + 16)(%rdx), RA3, RA3;
-	vpxor (3 * 32 + 16)(%rdx), RB0, RB0;
-	vpxor (4 * 32 + 16)(%rdx), RB1, RB1;
-	vpxor (5 * 32 + 16)(%rdx), RB2, RB2;
-	vpxor (6 * 32 + 16)(%rdx), RB3, RB3;
-	vmovdqu (7 * 32 + 16)(%rdx), RNOTx;
-	vmovdqu RNOTx, (%rcx); /* store new IV */
-
-	vmovdqu RA0, (0 * 32)(%rsi);
-	vmovdqu RA1, (1 * 32)(%rsi);
-	vmovdqu RA2, (2 * 32)(%rsi);
-	vmovdqu RA3, (3 * 32)(%rsi);
-	vmovdqu RB0, (4 * 32)(%rsi);
-	vmovdqu RB1, (5 * 32)(%rsi);
-	vmovdqu RB2, (6 * 32)(%rsi);
-	vmovdqu RB3, (7 * 32)(%rsi);
-
-	vzeroall;
-
-	ret_spec_stop;
-	CFI_ENDPROC();
-ELF(.size _gcry_sm4_gfni_avx2_cbc_dec,.-_gcry_sm4_gfni_avx2_cbc_dec;)
-
-.align 16
-.globl _gcry_sm4_gfni_avx2_cfb_dec
-ELF(.type   _gcry_sm4_gfni_avx2_cfb_dec, at function;)
-_gcry_sm4_gfni_avx2_cfb_dec:
-	/* input:
-	 *	%rdi: ctx, CTX
-	 *	%rsi: dst (16 blocks)
-	 *	%rdx: src (16 blocks)
-	 *	%rcx: iv
-	 */
-	CFI_STARTPROC();
-
-	/* Load input */
-	vmovdqu (%rcx), RNOTx;
-	vinserti128 $1, (%rdx), RNOT, RA0;
-	vmovdqu (0 * 32 + 16)(%rdx), RA1;
-	vmovdqu (1 * 32 + 16)(%rdx), RA2;
-	vmovdqu (2 * 32 + 16)(%rdx), RA3;
-	vmovdqu (3 * 32 + 16)(%rdx), RB0;
-	vmovdqu (4 * 32 + 16)(%rdx), RB1;
-	vmovdqu (5 * 32 + 16)(%rdx), RB2;
-	vmovdqu (6 * 32 + 16)(%rdx), RB3;
-
-	/* Update IV */
-	vmovdqu (7 * 32 + 16)(%rdx), RNOTx;
-	vmovdqu RNOTx, (%rcx);
-
-	call __sm4_gfni_crypt_blk16;
-
-	vpxor (0 * 32)(%rdx), RA0, RA0;
-	vpxor (1 * 32)(%rdx), RA1, RA1;
-	vpxor (2 * 32)(%rdx), RA2, RA2;
-	vpxor (3 * 32)(%rdx), RA3, RA3;
-	vpxor (4 * 32)(%rdx), RB0, RB0;
-	vpxor (5 * 32)(%rdx), RB1, RB1;
-	vpxor (6 * 32)(%rdx), RB2, RB2;
-	vpxor (7 * 32)(%rdx), RB3, RB3;
-
-	vmovdqu RA0, (0 * 32)(%rsi);
-	vmovdqu RA1, (1 * 32)(%rsi);
-	vmovdqu RA2, (2 * 32)(%rsi);
-	vmovdqu RA3, (3 * 32)(%rsi);
-	vmovdqu RB0, (4 * 32)(%rsi);
-	vmovdqu RB1, (5 * 32)(%rsi);
-	vmovdqu RB2, (6 * 32)(%rsi);
-	vmovdqu RB3, (7 * 32)(%rsi);
-
-	vzeroall;
-
-	ret_spec_stop;
-	CFI_ENDPROC();
-ELF(.size _gcry_sm4_gfni_avx2_cfb_dec,.-_gcry_sm4_gfni_avx2_cfb_dec;)
-
-.align 16
-.globl _gcry_sm4_gfni_avx2_ocb_enc
-ELF(.type _gcry_sm4_gfni_avx2_ocb_enc, at function;)
-
-_gcry_sm4_gfni_avx2_ocb_enc:
-	/* input:
-	 *	%rdi: ctx, CTX
-	 *	%rsi: dst (16 blocks)
-	 *	%rdx: src (16 blocks)
-	 *	%rcx: offset
-	 *	%r8 : checksum
-	 *	%r9 : L pointers (void *L[16])
-	 */
-	CFI_STARTPROC();
-
-	subq $(4 * 8), %rsp;
-	CFI_ADJUST_CFA_OFFSET(4 * 8);
-
-	movq %r10, (0 * 8)(%rsp);
-	movq %r11, (1 * 8)(%rsp);
-	movq %r12, (2 * 8)(%rsp);
-	movq %r13, (3 * 8)(%rsp);
-	CFI_REL_OFFSET(%r10, 0 * 8);
-	CFI_REL_OFFSET(%r11, 1 * 8);
-	CFI_REL_OFFSET(%r12, 2 * 8);
-	CFI_REL_OFFSET(%r13, 3 * 8);
-
-	vmovdqu (%rcx), RTMP0x;
-	vmovdqu (%r8), RTMP1x;
-
-	/* Offset_i = Offset_{i-1} xor L_{ntz(i)} */
-	/* Checksum_i = Checksum_{i-1} xor P_i  */
-	/* C_i = Offset_i xor ENCIPHER(K, P_i xor Offset_i)  */
-
-#define OCB_INPUT(n, l0reg, l1reg, yreg) \
-	  vmovdqu (n * 32)(%rdx), yreg; \
-	  vpxor (l0reg), RTMP0x, RNOTx; \
-	  vpxor (l1reg), RNOTx, RTMP0x; \
-	  vinserti128 $1, RTMP0x, RNOT, RNOT; \
-	  vpxor yreg, RTMP1, RTMP1; \
-	  vpxor yreg, RNOT, yreg; \
-	  vmovdqu RNOT, (n * 32)(%rsi);
-
-	movq (0 * 8)(%r9), %r10;
-	movq (1 * 8)(%r9), %r11;
-	movq (2 * 8)(%r9), %r12;
-	movq (3 * 8)(%r9), %r13;
-	OCB_INPUT(0, %r10, %r11, RA0);
-	OCB_INPUT(1, %r12, %r13, RA1);
-	movq (4 * 8)(%r9), %r10;
-	movq (5 * 8)(%r9), %r11;
-	movq (6 * 8)(%r9), %r12;
-	movq (7 * 8)(%r9), %r13;
-	OCB_INPUT(2, %r10, %r11, RA2);
-	OCB_INPUT(3, %r12, %r13, RA3);
-	movq (8 * 8)(%r9), %r10;
-	movq (9 * 8)(%r9), %r11;
-	movq (10 * 8)(%r9), %r12;
-	movq (11 * 8)(%r9), %r13;
-	OCB_INPUT(4, %r10, %r11, RB0);
-	OCB_INPUT(5, %r12, %r13, RB1);
-	movq (12 * 8)(%r9), %r10;
-	movq (13 * 8)(%r9), %r11;
-	movq (14 * 8)(%r9), %r12;
-	movq (15 * 8)(%r9), %r13;
-	OCB_INPUT(6, %r10, %r11, RB2);
-	OCB_INPUT(7, %r12, %r13, RB3);
-#undef OCB_INPUT
-
-	vextracti128 $1, RTMP1, RNOTx;
-	vmovdqu RTMP0x, (%rcx);
-	vpxor RNOTx, RTMP1x, RTMP1x;
-	vmovdqu RTMP1x, (%r8);
-
-	movq (0 * 8)(%rsp), %r10;
-	movq (1 * 8)(%rsp), %r11;
-	movq (2 * 8)(%rsp), %r12;
-	movq (3 * 8)(%rsp), %r13;
-	CFI_RESTORE(%r10);
-	CFI_RESTORE(%r11);
-	CFI_RESTORE(%r12);
-	CFI_RESTORE(%r13);
-
-	call __sm4_gfni_crypt_blk16;
-
-	addq $(4 * 8), %rsp;
-	CFI_ADJUST_CFA_OFFSET(-4 * 8);
-
-	vpxor (0 * 32)(%rsi), RA0, RA0;
-	vpxor (1 * 32)(%rsi), RA1, RA1;
-	vpxor (2 * 32)(%rsi), RA2, RA2;
-	vpxor (3 * 32)(%rsi), RA3, RA3;
-	vpxor (4 * 32)(%rsi), RB0, RB0;
-	vpxor (5 * 32)(%rsi), RB1, RB1;
-	vpxor (6 * 32)(%rsi), RB2, RB2;
-	vpxor (7 * 32)(%rsi), RB3, RB3;
-
-	vmovdqu RA0, (0 * 32)(%rsi);
-	vmovdqu RA1, (1 * 32)(%rsi);
-	vmovdqu RA2, (2 * 32)(%rsi);
-	vmovdqu RA3, (3 * 32)(%rsi);
-	vmovdqu RB0, (4 * 32)(%rsi);
-	vmovdqu RB1, (5 * 32)(%rsi);
-	vmovdqu RB2, (6 * 32)(%rsi);
-	vmovdqu RB3, (7 * 32)(%rsi);
-
-	vzeroall;
-
-	ret_spec_stop;
-	CFI_ENDPROC();
-ELF(.size _gcry_sm4_gfni_avx2_ocb_enc,.-_gcry_sm4_gfni_avx2_ocb_enc;)
-
-.align 16
-.globl _gcry_sm4_gfni_avx2_ocb_dec
-ELF(.type _gcry_sm4_gfni_avx2_ocb_dec, at function;)
-
-_gcry_sm4_gfni_avx2_ocb_dec:
-	/* input:
-	 *	%rdi: ctx, CTX
-	 *	%rsi: dst (16 blocks)
-	 *	%rdx: src (16 blocks)
-	 *	%rcx: offset
-	 *	%r8 : checksum
-	 *	%r9 : L pointers (void *L[16])
-	 */
-	CFI_STARTPROC();
-
-	subq $(4 * 8), %rsp;
-	CFI_ADJUST_CFA_OFFSET(4 * 8);
-
-	movq %r10, (0 * 8)(%rsp);
-	movq %r11, (1 * 8)(%rsp);
-	movq %r12, (2 * 8)(%rsp);
-	movq %r13, (3 * 8)(%rsp);
-	CFI_REL_OFFSET(%r10, 0 * 8);
-	CFI_REL_OFFSET(%r11, 1 * 8);
-	CFI_REL_OFFSET(%r12, 2 * 8);
-	CFI_REL_OFFSET(%r13, 3 * 8);
-
-	vmovdqu (%rcx), RTMP0x;
-
-	/* Offset_i = Offset_{i-1} xor L_{ntz(i)} */
-	/* C_i = Offset_i xor ENCIPHER(K, P_i xor Offset_i)  */
-
-#define OCB_INPUT(n, l0reg, l1reg, yreg) \
-	  vmovdqu (n * 32)(%rdx), yreg; \
-	  vpxor (l0reg), RTMP0x, RNOTx; \
-	  vpxor (l1reg), RNOTx, RTMP0x; \
-	  vinserti128 $1, RTMP0x, RNOT, RNOT; \
-	  vpxor yreg, RNOT, yreg; \
-	  vmovdqu RNOT, (n * 32)(%rsi);
-
-	movq (0 * 8)(%r9), %r10;
-	movq (1 * 8)(%r9), %r11;
-	movq (2 * 8)(%r9), %r12;
-	movq (3 * 8)(%r9), %r13;
-	OCB_INPUT(0, %r10, %r11, RA0);
-	OCB_INPUT(1, %r12, %r13, RA1);
-	movq (4 * 8)(%r9), %r10;
-	movq (5 * 8)(%r9), %r11;
-	movq (6 * 8)(%r9), %r12;
-	movq (7 * 8)(%r9), %r13;
-	OCB_INPUT(2, %r10, %r11, RA2);
-	OCB_INPUT(3, %r12, %r13, RA3);
-	movq (8 * 8)(%r9), %r10;
-	movq (9 * 8)(%r9), %r11;
-	movq (10 * 8)(%r9), %r12;
-	movq (11 * 8)(%r9), %r13;
-	OCB_INPUT(4, %r10, %r11, RB0);
-	OCB_INPUT(5, %r12, %r13, RB1);
-	movq (12 * 8)(%r9), %r10;
-	movq (13 * 8)(%r9), %r11;
-	movq (14 * 8)(%r9), %r12;
-	movq (15 * 8)(%r9), %r13;
-	OCB_INPUT(6, %r10, %r11, RB2);
-	OCB_INPUT(7, %r12, %r13, RB3);
-#undef OCB_INPUT
-
-	vmovdqu RTMP0x, (%rcx);
-
-	movq (0 * 8)(%rsp), %r10;
-	movq (1 * 8)(%rsp), %r11;
-	movq (2 * 8)(%rsp), %r12;
-	movq (3 * 8)(%rsp), %r13;
-	CFI_RESTORE(%r10);
-	CFI_RESTORE(%r11);
-	CFI_RESTORE(%r12);
-	CFI_RESTORE(%r13);
-
-	call __sm4_gfni_crypt_blk16;
-
-	addq $(4 * 8), %rsp;
-	CFI_ADJUST_CFA_OFFSET(-4 * 8);
-
-	vmovdqu (%r8), RTMP1x;
-
-	vpxor (0 * 32)(%rsi), RA0, RA0;
-	vpxor (1 * 32)(%rsi), RA1, RA1;
-	vpxor (2 * 32)(%rsi), RA2, RA2;
-	vpxor (3 * 32)(%rsi), RA3, RA3;
-	vpxor (4 * 32)(%rsi), RB0, RB0;
-	vpxor (5 * 32)(%rsi), RB1, RB1;
-	vpxor (6 * 32)(%rsi), RB2, RB2;
-	vpxor (7 * 32)(%rsi), RB3, RB3;
-
-	/* Checksum_i = Checksum_{i-1} xor P_i  */
-
-	vmovdqu RA0, (0 * 32)(%rsi);
-	vpxor RA0, RTMP1, RTMP1;
-	vmovdqu RA1, (1 * 32)(%rsi);
-	vpxor RA1, RTMP1, RTMP1;
-	vmovdqu RA2, (2 * 32)(%rsi);
-	vpxor RA2, RTMP1, RTMP1;
-	vmovdqu RA3, (3 * 32)(%rsi);
-	vpxor RA3, RTMP1, RTMP1;
-	vmovdqu RB0, (4 * 32)(%rsi);
-	vpxor RB0, RTMP1, RTMP1;
-	vmovdqu RB1, (5 * 32)(%rsi);
-	vpxor RB1, RTMP1, RTMP1;
-	vmovdqu RB2, (6 * 32)(%rsi);
-	vpxor RB2, RTMP1, RTMP1;
-	vmovdqu RB3, (7 * 32)(%rsi);
-	vpxor RB3, RTMP1, RTMP1;
-
-	vextracti128 $1, RTMP1, RNOTx;
-	vpxor RNOTx, RTMP1x, RTMP1x;
-	vmovdqu RTMP1x, (%r8);
-
-	vzeroall;
-
-	ret_spec_stop;
-	CFI_ENDPROC();
-ELF(.size _gcry_sm4_gfni_avx2_ocb_dec,.-_gcry_sm4_gfni_avx2_ocb_dec;)
-
-.align 16
-.globl _gcry_sm4_gfni_avx2_ocb_auth
-ELF(.type _gcry_sm4_gfni_avx2_ocb_auth, at function;)
-
-_gcry_sm4_gfni_avx2_ocb_auth:
-	/* input:
-	 *	%rdi: ctx, CTX
-	 *	%rsi: abuf (16 blocks)
-	 *	%rdx: offset
-	 *	%rcx: checksum
-	 *	%r8 : L pointers (void *L[16])
-	 */
-	CFI_STARTPROC();
-
-	subq $(4 * 8), %rsp;
-	CFI_ADJUST_CFA_OFFSET(4 * 8);
-
-	movq %r10, (0 * 8)(%rsp);
-	movq %r11, (1 * 8)(%rsp);
-	movq %r12, (2 * 8)(%rsp);
-	movq %r13, (3 * 8)(%rsp);
-	CFI_REL_OFFSET(%r10, 0 * 8);
-	CFI_REL_OFFSET(%r11, 1 * 8);
-	CFI_REL_OFFSET(%r12, 2 * 8);
-	CFI_REL_OFFSET(%r13, 3 * 8);
-
-	vmovdqu (%rdx), RTMP0x;
-
-	/* Offset_i = Offset_{i-1} xor L_{ntz(i)} */
-	/* Sum_i = Sum_{i-1} xor ENCIPHER(K, A_i xor Offset_i)  */
-
-#define OCB_INPUT(n, l0reg, l1reg, yreg) \
-	  vmovdqu (n * 32)(%rsi), yreg; \
-	  vpxor (l0reg), RTMP0x, RNOTx; \
-	  vpxor (l1reg), RNOTx, RTMP0x; \
-	  vinserti128 $1, RTMP0x, RNOT, RNOT; \
-	  vpxor yreg, RNOT, yreg;
-
-	movq (0 * 8)(%r8), %r10;
-	movq (1 * 8)(%r8), %r11;
-	movq (2 * 8)(%r8), %r12;
-	movq (3 * 8)(%r8), %r13;
-	OCB_INPUT(0, %r10, %r11, RA0);
-	OCB_INPUT(1, %r12, %r13, RA1);
-	movq (4 * 8)(%r8), %r10;
-	movq (5 * 8)(%r8), %r11;
-	movq (6 * 8)(%r8), %r12;
-	movq (7 * 8)(%r8), %r13;
-	OCB_INPUT(2, %r10, %r11, RA2);
-	OCB_INPUT(3, %r12, %r13, RA3);
-	movq (8 * 8)(%r8), %r10;
-	movq (9 * 8)(%r8), %r11;
-	movq (10 * 8)(%r8), %r12;
-	movq (11 * 8)(%r8), %r13;
-	OCB_INPUT(4, %r10, %r11, RB0);
-	OCB_INPUT(5, %r12, %r13, RB1);
-	movq (12 * 8)(%r8), %r10;
-	movq (13 * 8)(%r8), %r11;
-	movq (14 * 8)(%r8), %r12;
-	movq (15 * 8)(%r8), %r13;
-	OCB_INPUT(6, %r10, %r11, RB2);
-	OCB_INPUT(7, %r12, %r13, RB3);
-#undef OCB_INPUT
-
-	vmovdqu RTMP0x, (%rdx);
-
-	movq (0 * 8)(%rsp), %r10;
-	movq (1 * 8)(%rsp), %r11;
-	movq (2 * 8)(%rsp), %r12;
-	movq (3 * 8)(%rsp), %r13;
-	CFI_RESTORE(%r10);
-	CFI_RESTORE(%r11);
-	CFI_RESTORE(%r12);
-	CFI_RESTORE(%r13);
-
-	call __sm4_gfni_crypt_blk16;
-
-	addq $(4 * 8), %rsp;
-	CFI_ADJUST_CFA_OFFSET(-4 * 8);
-
-	vpxor RA0, RB0, RA0;
-	vpxor RA1, RB1, RA1;
-	vpxor RA2, RB2, RA2;
-	vpxor RA3, RB3, RA3;
-
-	vpxor RA1, RA0, RA0;
-	vpxor RA3, RA2, RA2;
-
-	vpxor RA2, RA0, RTMP1;
-
-	vextracti128 $1, RTMP1, RNOTx;
-	vpxor (%rcx), RTMP1x, RTMP1x;
-	vpxor RNOTx, RTMP1x, RTMP1x;
-	vmovdqu RTMP1x, (%rcx);
-
-	vzeroall;
-
-	ret_spec_stop;
-	CFI_ENDPROC();
-ELF(.size _gcry_sm4_gfni_avx2_ocb_auth,.-_gcry_sm4_gfni_avx2_ocb_auth;)
+#define FUNC_NAME(func) _gcry_sm4_gfni_avx2_ ## func
+#define SM4_CRYPT_BLK16 __sm4_gfni_crypt_blk16
+#include "sm4-avx2-amd64.h"
 
 #endif /*defined(ENABLE_GFNI_SUPPORT) && defined(ENABLE_AVX2_SUPPORT)*/
 #endif /*__x86_64*/
-- 
2.53.0



From jussi.kivilinna at iki.fi  Sun Jun 28 14:37:40 2026
From: jussi.kivilinna at iki.fi (Jussi Kivilinna)
Date: Sun, 28 Jun 2026 15:37:40 +0300
Subject: [PATCH 6/7] sm4: add Intel SM4 instructions accelerated AVX512
 implementation
In-Reply-To: <20260628123742.1916-1-jussi.kivilinna@iki.fi>
References: <20260628123742.1916-1-jussi.kivilinna@iki.fi>
Message-ID: <20260628123742.1916-6-jussi.kivilinna@iki.fi>

* cipher/Makefile.am: Add 'sm4-avx512-amd64.h' and
'sm4-intel-avx512-amd64.S'.
* cipher/sm4-avx512-amd64.h: New, 32-block cipher mode functions moved
here from 'sm4-gfni-avx512-amd64.S'.
* cipher/sm4-gfni-avx512-amd64.S (_gcry_sm4_gfni_avx512_crypt_blk32)
(_gcry_sm4_gfni_avx512_ctr_enc_blk32)
(_gcry_sm4_gfni_avx512_cbc_dec_blk32)
(_gcry_sm4_gfni_avx512_cfb_dec_blk32)
(_gcry_sm4_gfni_avx512_ocb_enc_blk32)
(_gcry_sm4_gfni_avx512_ocb_dec_blk32): Move to 'sm4-avx512-amd64.h'.
(FUNC_NAME, SM4_CRYPT_BLK32): New.
* cipher/sm4-intel-avx512-amd64.S: New.
* cipher/sm4.c (USE_INTEL_SM4_AVX512): New.
(ASM_FUNC_ABI): Define also for Intel SM4 AVX512 implementation.
(SM4_context): Add 'use_intel_sm4_avx512'.
(_gcry_sm4_intel_avx512_ctr_enc_blk32)
(_gcry_sm4_intel_avx512_cbc_dec_blk32)
(_gcry_sm4_intel_avx512_cfb_dec_blk32)
(_gcry_sm4_intel_avx512_ocb_enc_blk32)
(_gcry_sm4_intel_avx512_ocb_dec_blk32)
(_gcry_sm4_intel_avx512_crypt_blk32): New.
(sm4_setkey): Enable Intel SM4 AVX512 implementation if supported by CPU.
(sm4_crypt_blk1_32, _gcry_sm4_ctr_enc, _gcry_sm4_cbc_dec)
(_gcry_sm4_cfb_dec, _gcry_sm4_ocb_crypt) [USE_INTEL_SM4_AVX512]: Add
Intel SM4 AVX512 code path.
* configure.ac (GCRYPT_ASM_CIPHERS) [x86_64]: Add
'sm4-intel-avx512-amd64.lo'.
--

Add a 32-block SM4 implementation using the Intel SM4 instruction set
extension (EVEX/512-bit encoded VSM4RNDS4).  The 32-block cipher mode
code is shared with the GFNI/AVX512 implementation through a new
'sm4-avx512-amd64.h' header; the key schedule and the 16-block and
shorter paths are reused from the AVX2 implementation.  It is used in
preference to the GFNI/AVX512 implementation when available.

Tested with Intel SDE; not yet tested on real hardware.

Signed-off-by: Jussi Kivilinna <jussi.kivilinna at iki.fi>
---
 cipher/Makefile.am              |   4 +-
 cipher/sm4-avx512-amd64.h       | 620 ++++++++++++++++++++++++++++++++
 cipher/sm4-gfni-avx512-amd64.S  | 593 +-----------------------------
 cipher/sm4-intel-avx512-amd64.S | 174 +++++++++
 cipher/sm4.c                    | 134 ++++++-
 configure.ac                    |   1 +
 6 files changed, 934 insertions(+), 592 deletions(-)
 create mode 100644 cipher/sm4-avx512-amd64.h
 create mode 100644 cipher/sm4-intel-avx512-amd64.S

diff --git a/cipher/Makefile.am b/cipher/Makefile.am
index 04094a10..8f54ea47 100644
--- a/cipher/Makefile.am
+++ b/cipher/Makefile.am
@@ -137,8 +137,8 @@ EXTRA_libcipher_la_SOURCES = \
 	serpent-avx512-x86.c serpent-armv7-neon.S \
 	simd-common-aarch64.h simd-common-ppc.h simd-common-riscv.h \
 	sm4.c sm4-aesni-avx-amd64.S sm4-aesni-avx2-amd64.S \
-	sm4-avx2-amd64.h \
-	sm4-intel-avx2-amd64.S \
+	sm4-avx2-amd64.h sm4-avx512-amd64.h \
+	sm4-intel-avx2-amd64.S sm4-intel-avx512-amd64.S \
 	sm4-gfni-avx2-amd64.S sm4-gfni-avx512-amd64.S \
 	sm4-aarch64.S sm4-armv8-aarch64-ce.S sm4-armv9-aarch64-sve-ce.S \
 	sm4-ppc.c \
diff --git a/cipher/sm4-avx512-amd64.h b/cipher/sm4-avx512-amd64.h
new file mode 100644
index 00000000..3ee5a4f6
--- /dev/null
+++ b/cipher/sm4-avx512-amd64.h
@@ -0,0 +1,620 @@
+/* sm4-avx512-amd64.h  -  Shared AVX512 32-block cipher mode code for SM4
+ *
+ * Copyright (C) 2022-2023 Jussi Kivilinna <jussi.kivilinna at iki.fi>
+ *
+ * This file is part of Libgcrypt.
+ *
+ * Libgcrypt is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * Libgcrypt is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef GCRY_SM4_AVX512_AMD64_H
+#define GCRY_SM4_AVX512_AMD64_H
+
+SECTION_RODATA
+.align 64
+
+ELF(.type FUNC_NAME(cipher_mode_consts), at object)
+FUNC_NAME(cipher_mode_consts):
+
+/* CTR mode addition constants */
+
+.Lcounter0123_lo:
+	.quad 0, 0
+	.quad 1, 0
+	.quad 2, 0
+	.quad 3, 0
+
+.Lbige_addb_0_1:
+	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1
+.Lbige_addb_2_3:
+	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2
+	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3
+.Lbige_addb_4_5:
+	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4
+	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5
+.Lbige_addb_6_7:
+	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6
+	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7
+.Lbige_addb_8_9:
+	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8
+	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9
+.Lbige_addb_10_11:
+	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10
+	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11
+.Lbige_addb_12_13:
+	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12
+	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13
+.Lbige_addb_14_15:
+	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14
+	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15
+.Lbige_addb_16:
+	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16
+
+.Lcounter2222_lo:
+	.quad 2, 0
+.Lcounter4444_lo:
+	.quad 4, 0
+.Lcounter8888_lo:
+	.quad 8, 0
+.Lcounter16161616_lo:
+	.quad 16, 0
+.Lcounter1111_hi:
+	.quad 0, 1
+
+.text
+
+.align 16
+.globl FUNC_NAME(crypt_blk32)
+ELF(.type   FUNC_NAME(crypt_blk32), at function;)
+FUNC_NAME(crypt_blk32):
+	/* input:
+	 *	%rdi: ctx, CTX
+	 *	%rsi: dst (32 blocks)
+	 *	%rdx: src (32 blocks)
+	 */
+	CFI_STARTPROC();
+	spec_stop_avx512;
+
+	/* Load input */
+	vmovdqu32 (0 * 64)(%rdx), RA0z;
+	vmovdqu32 (1 * 64)(%rdx), RA1z;
+	vmovdqu32 (2 * 64)(%rdx), RA2z;
+	vmovdqu32 (3 * 64)(%rdx), RA3z;
+	vmovdqu32 (4 * 64)(%rdx), RB0z;
+	vmovdqu32 (5 * 64)(%rdx), RB1z;
+	vmovdqu32 (6 * 64)(%rdx), RB2z;
+	vmovdqu32 (7 * 64)(%rdx), RB3z;
+
+	call SM4_CRYPT_BLK32;
+
+	vmovdqu32 RA0z, (0 * 64)(%rsi);
+	vmovdqu32 RA1z, (1 * 64)(%rsi);
+	vmovdqu32 RA2z, (2 * 64)(%rsi);
+	vmovdqu32 RA3z, (3 * 64)(%rsi);
+	vmovdqu32 RB0z, (4 * 64)(%rsi);
+	vmovdqu32 RB1z, (5 * 64)(%rsi);
+	vmovdqu32 RB2z, (6 * 64)(%rsi);
+	vmovdqu32 RB3z, (7 * 64)(%rsi);
+
+	xorl %eax, %eax;
+	vzeroall;
+
+	ret_spec_stop;
+	CFI_ENDPROC();
+ELF(.size FUNC_NAME(crypt_blk32),.-FUNC_NAME(crypt_blk32);)
+
+.align 16
+.globl FUNC_NAME(ctr_enc_blk32)
+ELF(.type   FUNC_NAME(ctr_enc_blk32), at function;)
+FUNC_NAME(ctr_enc_blk32):
+	/* input:
+	 *	%rdi: ctx, CTX
+	 *	%rsi: dst (32 blocks)
+	 *	%rdx: src (32 blocks)
+	 *	%rcx: iv (big endian, 128bit)
+	 */
+	CFI_STARTPROC();
+	spec_stop_avx512;
+
+	cmpb $(0x100 - 32), 15(%rcx);
+	jbe .Lctr_byteadd32;
+
+	vbroadcasti64x2 .Lbswap128_mask rRIP, RTMP0z;
+	vmovdqa32 .Lcounter0123_lo rRIP, RTMP1z;
+	vbroadcasti64x2 .Lcounter4444_lo rRIP, RTMP2z;
+	vbroadcasti64x2 .Lcounter8888_lo rRIP, RTMP3z;
+	vbroadcasti64x2 .Lcounter16161616_lo rRIP, RTMP4z;
+
+	/* load IV and byteswap */
+	movq 8(%rcx), %r11;
+	bswapq %r11;
+	vbroadcasti64x2 (%rcx), RB3z;
+	vpshufb RTMP0z, RB3z, RB3z;
+
+	/* check need for handling 64-bit overflow and carry */
+	cmpq $(0xffffffffffffffff - 32), %r11;
+	ja .Lhandle_ctr_carry_blk32;
+
+	/* construct IVs */
+	vpaddq RTMP1z, RB3z, RA0z; /* +0:+1:+2:+3 */
+	vpaddq RTMP2z, RA0z, RA1z; /* +4:+5:+6:+7 */
+	vpaddq RTMP3z, RA0z, RA2z; /* +8:+9:+10:+11 */
+	vpaddq RTMP3z, RA1z, RA3z; /* +12:+13:+14:+15 */
+	vpaddq RTMP4z, RA0z, RB0z; /* +16... */
+	vpaddq RTMP4z, RA1z, RB1z; /* +20... */
+	vpaddq RTMP4z, RA2z, RB2z; /* +24... */
+	vpaddq RTMP4z, RA3z, RB3z; /* +28... */
+
+	/* Update counter */
+	leaq 32(%r11), %r11;
+	bswapq %r11;
+	movq %r11, 8(%rcx);
+
+	jmp .Lctr_carry_done_blk32;
+
+.Lhandle_ctr_carry_blk32:
+	vbroadcasti64x2 .Lcounter1111_hi rRIP, RNOTz;
+
+	/* construct IVs */
+	add_le128(RA0z, RB3z, RTMP1z, RNOTz); /* +0:+1:+2:+3 */
+	add_le128(RA1z, RA0z, RTMP2z, RNOTz); /* +4:+5:+6:+7 */
+	add_le128(RA2z, RA0z, RTMP3z, RNOTz); /* +8:+9:+10:+11 */
+	add_le128(RA3z, RA1z, RTMP3z, RNOTz); /* +12:+13:+14:+15 */
+	add_le128(RB0z, RA0z, RTMP4z, RNOTz); /* +16... */
+	add_le128(RB1z, RA1z, RTMP4z, RNOTz); /* +20... */
+	add_le128(RB2z, RA2z, RTMP4z, RNOTz); /* +24... */
+	add_le128(RB3z, RA3z, RTMP4z, RNOTz); /* +28... */
+
+	/* Update counter */
+	addq $32, %r11;
+	movq (%rcx), %r10;
+	bswapq %r10;
+	adcq $0, %r10;
+	bswapq %r11;
+	bswapq %r10;
+	movq %r11, 8(%rcx);
+	movq %r10, (%rcx);
+
+.align 16
+.Lctr_carry_done_blk32:
+	/* Byte-swap IVs. */
+	vpshufb RTMP0z, RA0z, RA0z;
+	vpshufb RTMP0z, RA1z, RA1z;
+	vpshufb RTMP0z, RA2z, RA2z;
+	vpshufb RTMP0z, RA3z, RA3z;
+	vpshufb RTMP0z, RB0z, RB0z;
+	vpshufb RTMP0z, RB1z, RB1z;
+	vpshufb RTMP0z, RB2z, RB2z;
+	vpshufb RTMP0z, RB3z, RB3z;
+
+.align 16
+.Lload_ctr_done32:
+	call SM4_CRYPT_BLK32;
+
+	vpxord (0 * 64)(%rdx), RA0z, RA0z;
+	vpxord (1 * 64)(%rdx), RA1z, RA1z;
+	vpxord (2 * 64)(%rdx), RA2z, RA2z;
+	vpxord (3 * 64)(%rdx), RA3z, RA3z;
+	vpxord (4 * 64)(%rdx), RB0z, RB0z;
+	vpxord (5 * 64)(%rdx), RB1z, RB1z;
+	vpxord (6 * 64)(%rdx), RB2z, RB2z;
+	vpxord (7 * 64)(%rdx), RB3z, RB3z;
+
+	vmovdqu32 RA0z, (0 * 64)(%rsi);
+	vmovdqu32 RA1z, (1 * 64)(%rsi);
+	vmovdqu32 RA2z, (2 * 64)(%rsi);
+	vmovdqu32 RA3z, (3 * 64)(%rsi);
+	vmovdqu32 RB0z, (4 * 64)(%rsi);
+	vmovdqu32 RB1z, (5 * 64)(%rsi);
+	vmovdqu32 RB2z, (6 * 64)(%rsi);
+	vmovdqu32 RB3z, (7 * 64)(%rsi);
+
+	vzeroall;
+	kxorq %k1, %k1, %k1;
+
+	ret_spec_stop;
+
+.align 16
+.Lctr_byteadd_full_ctr_carry32:
+	movq 8(%rcx), %r11;
+	movq (%rcx), %r10;
+	bswapq %r11;
+	bswapq %r10;
+	addq $32, %r11;
+	adcq $0, %r10;
+	bswapq %r11;
+	bswapq %r10;
+	movq %r11, 8(%rcx);
+	movq %r10, (%rcx);
+	jmp .Lctr_byteadd_zmm32;
+.align 16
+.Lctr_byteadd32:
+	vbroadcasti64x2 (%rcx), RA3z;
+	je .Lctr_byteadd_full_ctr_carry32;
+	addb $32, 15(%rcx);
+.Lctr_byteadd_zmm32:
+	vbroadcasti64x2 .Lbige_addb_16 rRIP, RB3z;
+	vpaddb RB3z, RA3z, RB3z;
+	vpaddb .Lbige_addb_0_1 rRIP, RA3z, RA0z;
+	vpaddb .Lbige_addb_4_5 rRIP, RA3z, RA1z;
+	vpaddb .Lbige_addb_8_9 rRIP, RA3z, RA2z;
+	vpaddb .Lbige_addb_12_13 rRIP, RA3z, RA3z;
+	vpaddb .Lbige_addb_0_1 rRIP, RB3z, RB0z;
+	vpaddb .Lbige_addb_4_5 rRIP, RB3z, RB1z;
+	vpaddb .Lbige_addb_8_9 rRIP, RB3z, RB2z;
+	vpaddb .Lbige_addb_12_13 rRIP, RB3z, RB3z;
+
+	jmp .Lload_ctr_done32;
+	CFI_ENDPROC();
+ELF(.size FUNC_NAME(ctr_enc_blk32),.-FUNC_NAME(ctr_enc_blk32);)
+
+.align 16
+.globl FUNC_NAME(cbc_dec_blk32)
+ELF(.type   FUNC_NAME(cbc_dec_blk32), at function;)
+FUNC_NAME(cbc_dec_blk32):
+	/* input:
+	 *	%rdi: ctx, CTX
+	 *	%rsi: dst (32 blocks)
+	 *	%rdx: src (32 blocks)
+	 *	%rcx: iv
+	 */
+	CFI_STARTPROC();
+	spec_stop_avx512;
+
+	vmovdqu32 (0 * 64)(%rdx), RA0z;
+	vmovdqu32 (1 * 64)(%rdx), RA1z;
+	vmovdqu32 (2 * 64)(%rdx), RA2z;
+	vmovdqu32 (3 * 64)(%rdx), RA3z;
+	vmovdqu32 (4 * 64)(%rdx), RB0z;
+	vmovdqu32 (5 * 64)(%rdx), RB1z;
+	vmovdqu32 (6 * 64)(%rdx), RB2z;
+	vmovdqu32 (7 * 64)(%rdx), RB3z;
+
+	call SM4_CRYPT_BLK32;
+
+	vmovdqu (%rcx), RNOTx;
+	vinserti64x2 $1, (0 * 16)(%rdx), RNOT, RNOT;
+	vinserti64x4 $1, (1 * 16)(%rdx), RNOTz, RNOTz;
+	vpxord RNOTz, RA0z, RA0z;
+	vpxord (0 * 64 + 48)(%rdx), RA1z, RA1z;
+	vpxord (1 * 64 + 48)(%rdx), RA2z, RA2z;
+	vpxord (2 * 64 + 48)(%rdx), RA3z, RA3z;
+	vpxord (3 * 64 + 48)(%rdx), RB0z, RB0z;
+	vpxord (4 * 64 + 48)(%rdx), RB1z, RB1z;
+	vpxord (5 * 64 + 48)(%rdx), RB2z, RB2z;
+	vpxord (6 * 64 + 48)(%rdx), RB3z, RB3z;
+	vmovdqu (7 * 64 + 48)(%rdx), RNOTx;
+	vmovdqu RNOTx, (%rcx); /* store new IV */
+
+	vmovdqu32 RA0z, (0 * 64)(%rsi);
+	vmovdqu32 RA1z, (1 * 64)(%rsi);
+	vmovdqu32 RA2z, (2 * 64)(%rsi);
+	vmovdqu32 RA3z, (3 * 64)(%rsi);
+	vmovdqu32 RB0z, (4 * 64)(%rsi);
+	vmovdqu32 RB1z, (5 * 64)(%rsi);
+	vmovdqu32 RB2z, (6 * 64)(%rsi);
+	vmovdqu32 RB3z, (7 * 64)(%rsi);
+
+	vzeroall;
+
+	ret_spec_stop;
+	CFI_ENDPROC();
+ELF(.size FUNC_NAME(cbc_dec_blk32),.-FUNC_NAME(cbc_dec_blk32);)
+
+.align 16
+.globl FUNC_NAME(cfb_dec_blk32)
+ELF(.type   FUNC_NAME(cfb_dec_blk32), at function;)
+FUNC_NAME(cfb_dec_blk32):
+	/* input:
+	 *	%rdi: ctx, CTX
+	 *	%rsi: dst (32 blocks)
+	 *	%rdx: src (32 blocks)
+	 *	%rcx: iv
+	 */
+	CFI_STARTPROC();
+	spec_stop_avx512;
+
+	/* Load input */
+	vmovdqu (%rcx), RA0x;
+	vinserti64x2 $1, (%rdx), RA0, RA0;
+	vinserti64x4 $1, 16(%rdx), RA0z, RA0z;
+	vmovdqu32 (0 * 64 + 48)(%rdx), RA1z;
+	vmovdqu32 (1 * 64 + 48)(%rdx), RA2z;
+	vmovdqu32 (2 * 64 + 48)(%rdx), RA3z;
+	vmovdqu32 (3 * 64 + 48)(%rdx), RB0z;
+	vmovdqu32 (4 * 64 + 48)(%rdx), RB1z;
+	vmovdqu32 (5 * 64 + 48)(%rdx), RB2z;
+	vmovdqu32 (6 * 64 + 48)(%rdx), RB3z;
+
+	/* Update IV */
+	vmovdqu (7 * 64 + 48)(%rdx), RNOTx;
+	vmovdqu RNOTx, (%rcx);
+
+	call SM4_CRYPT_BLK32;
+
+	vpxord (0 * 64)(%rdx), RA0z, RA0z;
+	vpxord (1 * 64)(%rdx), RA1z, RA1z;
+	vpxord (2 * 64)(%rdx), RA2z, RA2z;
+	vpxord (3 * 64)(%rdx), RA3z, RA3z;
+	vpxord (4 * 64)(%rdx), RB0z, RB0z;
+	vpxord (5 * 64)(%rdx), RB1z, RB1z;
+	vpxord (6 * 64)(%rdx), RB2z, RB2z;
+	vpxord (7 * 64)(%rdx), RB3z, RB3z;
+
+	vmovdqu32 RA0z, (0 * 64)(%rsi);
+	vmovdqu32 RA1z, (1 * 64)(%rsi);
+	vmovdqu32 RA2z, (2 * 64)(%rsi);
+	vmovdqu32 RA3z, (3 * 64)(%rsi);
+	vmovdqu32 RB0z, (4 * 64)(%rsi);
+	vmovdqu32 RB1z, (5 * 64)(%rsi);
+	vmovdqu32 RB2z, (6 * 64)(%rsi);
+	vmovdqu32 RB3z, (7 * 64)(%rsi);
+
+	vzeroall;
+
+	ret_spec_stop;
+	CFI_ENDPROC();
+ELF(.size FUNC_NAME(cfb_dec_blk32),.-FUNC_NAME(cfb_dec_blk32);)
+
+.align 16
+.globl FUNC_NAME(ocb_enc_blk32)
+ELF(.type FUNC_NAME(ocb_enc_blk32), at function;)
+FUNC_NAME(ocb_enc_blk32):
+	/* input:
+	 *	%rdi: ctx, CTX
+	 *	%rsi: dst (32 blocks)
+	 *	%rdx: src (32 blocks)
+	 *	%rcx: offset
+	 *	%r8 : checksum
+	 *	%r9 : L pointers (void *L[32])
+	 */
+	CFI_STARTPROC();
+	spec_stop_avx512;
+
+	subq $(5 * 8), %rsp;
+	CFI_ADJUST_CFA_OFFSET(5 * 8);
+
+	movq %r12, (0 * 8)(%rsp);
+	movq %r13, (1 * 8)(%rsp);
+	movq %r14, (2 * 8)(%rsp);
+	movq %r15, (3 * 8)(%rsp);
+	movq %rbx, (4 * 8)(%rsp);
+	CFI_REL_OFFSET(%r12, 0 * 8);
+	CFI_REL_OFFSET(%r13, 1 * 8);
+	CFI_REL_OFFSET(%r14, 2 * 8);
+	CFI_REL_OFFSET(%r15, 3 * 8);
+	CFI_REL_OFFSET(%rbx, 4 * 8);
+
+	vmovdqu (%rcx), RTMP0x;
+
+	/* Offset_i = Offset_{i-1} xor L_{ntz(i)} */
+	/* Checksum_i = Checksum_{i-1} xor P_i  */
+	/* C_i = Offset_i xor ENCIPHER(K, P_i xor Offset_i)  */
+
+#define OCB_INPUT(n, l0reg, l1reg, l2reg, l3reg, zreg, zplain) \
+	  vmovdqu32 (n * 64)(%rdx), zplain; \
+	  vpxor (l0reg), RTMP0x, RNOTx; \
+	  vpxor (l1reg), RNOTx, RTMP0x; \
+	  vinserti64x2 $1, RTMP0x, RNOT, RNOT; \
+	  vpxor (l2reg), RTMP0x, RTMP0x; \
+	  vinserti64x2 $2, RTMP0x, RNOTz, RNOTz; \
+	  vpxor (l3reg), RTMP0x, RTMP0x; \
+	  vinserti64x2 $3, RTMP0x, RNOTz, RNOTz; \
+	  vpxord zplain, RNOTz, zreg; \
+	  vmovdqu32 RNOTz, (n * 64)(%rsi);
+
+#define OCB_LOAD_PTRS(n) \
+	  movq ((n * 4 * 8) + (0 * 8))(%r9), %r10; \
+	  movq ((n * 4 * 8) + (1 * 8))(%r9), %r11; \
+	  movq ((n * 4 * 8) + (2 * 8))(%r9), %r12; \
+	  movq ((n * 4 * 8) + (3 * 8))(%r9), %r13; \
+	  movq ((n * 4 * 8) + (4 * 8))(%r9), %r14; \
+	  movq ((n * 4 * 8) + (5 * 8))(%r9), %r15; \
+	  movq ((n * 4 * 8) + (6 * 8))(%r9), %rax; \
+	  movq ((n * 4 * 8) + (7 * 8))(%r9), %rbx;
+
+	OCB_LOAD_PTRS(0);
+	OCB_INPUT(0, %r10, %r11, %r12, %r13, RA0z, RTMP1z);
+	OCB_INPUT(1, %r14, %r15, %rax, %rbx, RA1z, RTMP2z);
+	OCB_LOAD_PTRS(2);
+	OCB_INPUT(2, %r10, %r11, %r12, %r13, RA2z, RTMP3z);
+	vpternlogd $0x96, RTMP1z, RTMP2z, RTMP3z;
+	OCB_INPUT(3, %r14, %r15, %rax, %rbx, RA3z, RTMP4z);
+	OCB_LOAD_PTRS(4);
+	OCB_INPUT(4, %r10, %r11, %r12, %r13, RB0z, RX0z);
+	OCB_INPUT(5, %r14, %r15, %rax, %rbx, RB1z, RX1z);
+	vpternlogd $0x96, RTMP4z, RX0z, RX1z;
+	OCB_LOAD_PTRS(6);
+	OCB_INPUT(6, %r10, %r11, %r12, %r13, RB2z, RTMP4z);
+	OCB_INPUT(7, %r14, %r15, %rax, %rbx, RB3z, RX0z);
+#undef OCB_LOAD_PTRS
+#undef OCB_INPUT
+
+	vpternlogd $0x96, RTMP3z, RTMP4z, RX0z;
+	vpxord RX1z, RX0z, RNOTz;
+	vextracti64x4 $1, RNOTz, RTMP1;
+	vpxor RTMP1, RNOT, RNOT;
+	vextracti128 $1, RNOT, RTMP1x;
+	vpternlogd $0x96, (%r8), RTMP1x, RNOTx;
+
+	movq (0 * 8)(%rsp), %r12;
+	movq (1 * 8)(%rsp), %r13;
+	movq (2 * 8)(%rsp), %r14;
+	movq (3 * 8)(%rsp), %r15;
+	movq (4 * 8)(%rsp), %rbx;
+	CFI_RESTORE(%r12);
+	CFI_RESTORE(%r13);
+	CFI_RESTORE(%r14);
+	CFI_RESTORE(%r15);
+	CFI_RESTORE(%rbx);
+
+	vmovdqu RTMP0x, (%rcx);
+	vmovdqu RNOTx, (%r8);
+
+	call SM4_CRYPT_BLK32;
+
+	addq $(5 * 8), %rsp;
+	CFI_ADJUST_CFA_OFFSET(-5 * 8);
+
+	vpxord (0 * 64)(%rsi), RA0z, RA0z;
+	vpxord (1 * 64)(%rsi), RA1z, RA1z;
+	vpxord (2 * 64)(%rsi), RA2z, RA2z;
+	vpxord (3 * 64)(%rsi), RA3z, RA3z;
+	vpxord (4 * 64)(%rsi), RB0z, RB0z;
+	vpxord (5 * 64)(%rsi), RB1z, RB1z;
+	vpxord (6 * 64)(%rsi), RB2z, RB2z;
+	vpxord (7 * 64)(%rsi), RB3z, RB3z;
+
+	vmovdqu32 RA0z, (0 * 64)(%rsi);
+	vmovdqu32 RA1z, (1 * 64)(%rsi);
+	vmovdqu32 RA2z, (2 * 64)(%rsi);
+	vmovdqu32 RA3z, (3 * 64)(%rsi);
+	vmovdqu32 RB0z, (4 * 64)(%rsi);
+	vmovdqu32 RB1z, (5 * 64)(%rsi);
+	vmovdqu32 RB2z, (6 * 64)(%rsi);
+	vmovdqu32 RB3z, (7 * 64)(%rsi);
+
+	vzeroall;
+
+	ret_spec_stop;
+	CFI_ENDPROC();
+ELF(.size FUNC_NAME(ocb_enc_blk32),.-FUNC_NAME(ocb_enc_blk32);)
+
+.align 16
+.globl FUNC_NAME(ocb_dec_blk32)
+ELF(.type FUNC_NAME(ocb_dec_blk32), at function;)
+FUNC_NAME(ocb_dec_blk32):
+	/* input:
+	 *	%rdi: ctx, CTX
+	 *	%rsi: dst (32 blocks)
+	 *	%rdx: src (32 blocks)
+	 *	%rcx: offset
+	 *	%r8 : checksum
+	 *	%r9 : L pointers (void *L[32])
+	 */
+	CFI_STARTPROC();
+	spec_stop_avx512;
+
+	subq $(5 * 8), %rsp;
+	CFI_ADJUST_CFA_OFFSET(5 * 8);
+
+	movq %r12, (0 * 8)(%rsp);
+	movq %r13, (1 * 8)(%rsp);
+	movq %r14, (2 * 8)(%rsp);
+	movq %r15, (3 * 8)(%rsp);
+	movq %rbx, (4 * 8)(%rsp);
+	CFI_REL_OFFSET(%r12, 0 * 8);
+	CFI_REL_OFFSET(%r13, 1 * 8);
+	CFI_REL_OFFSET(%r14, 2 * 8);
+	CFI_REL_OFFSET(%r15, 3 * 8);
+	CFI_REL_OFFSET(%rbx, 4 * 8);
+
+	vmovdqu (%rcx), RTMP0x;
+
+	/* Offset_i = Offset_{i-1} xor L_{ntz(i)} */
+	/* C_i = Offset_i xor DECIPHER(K, P_i xor Offset_i)  */
+
+#define OCB_INPUT(n, l0reg, l1reg, l2reg, l3reg, zreg) \
+	  vmovdqu32 (n * 64)(%rdx), RTMP1z; \
+	  vpxor (l0reg), RTMP0x, RNOTx; \
+	  vpxor (l1reg), RNOTx, RTMP0x; \
+	  vinserti64x2 $1, RTMP0x, RNOT, RNOT; \
+	  vpxor (l2reg), RTMP0x, RTMP0x; \
+	  vinserti64x2 $2, RTMP0x, RNOTz, RNOTz; \
+	  vpxor (l3reg), RTMP0x, RTMP0x; \
+	  vinserti64x2 $3, RTMP0x, RNOTz, RNOTz; \
+	  vpxord RTMP1z, RNOTz, zreg; \
+	  vmovdqu32 RNOTz, (n * 64)(%rsi);
+
+#define OCB_LOAD_PTRS(n) \
+	  movq ((n * 4 * 8) + (0 * 8))(%r9), %r10; \
+	  movq ((n * 4 * 8) + (1 * 8))(%r9), %r11; \
+	  movq ((n * 4 * 8) + (2 * 8))(%r9), %r12; \
+	  movq ((n * 4 * 8) + (3 * 8))(%r9), %r13; \
+	  movq ((n * 4 * 8) + (4 * 8))(%r9), %r14; \
+	  movq ((n * 4 * 8) + (5 * 8))(%r9), %r15; \
+	  movq ((n * 4 * 8) + (6 * 8))(%r9), %rax; \
+	  movq ((n * 4 * 8) + (7 * 8))(%r9), %rbx;
+
+	OCB_LOAD_PTRS(0);
+	OCB_INPUT(0, %r10, %r11, %r12, %r13, RA0z);
+	OCB_INPUT(1, %r14, %r15, %rax, %rbx, RA1z);
+	OCB_LOAD_PTRS(2);
+	OCB_INPUT(2, %r10, %r11, %r12, %r13, RA2z);
+	OCB_INPUT(3, %r14, %r15, %rax, %rbx, RA3z);
+	OCB_LOAD_PTRS(4);
+	OCB_INPUT(4, %r10, %r11, %r12, %r13, RB0z);
+	OCB_INPUT(5, %r14, %r15, %rax, %rbx, RB1z);
+	OCB_LOAD_PTRS(6);
+	OCB_INPUT(6, %r10, %r11, %r12, %r13, RB2z);
+	OCB_INPUT(7, %r14, %r15, %rax, %rbx, RB3z);
+#undef OCB_LOAD_PTRS
+#undef OCB_INPUT
+
+	movq (0 * 8)(%rsp), %r12;
+	movq (1 * 8)(%rsp), %r13;
+	movq (2 * 8)(%rsp), %r14;
+	movq (3 * 8)(%rsp), %r15;
+	movq (4 * 8)(%rsp), %rbx;
+	CFI_RESTORE(%r12);
+	CFI_RESTORE(%r13);
+	CFI_RESTORE(%r14);
+	CFI_RESTORE(%r15);
+	CFI_RESTORE(%rbx);
+
+	vmovdqu RTMP0x, (%rcx);
+
+	call SM4_CRYPT_BLK32;
+
+	addq $(5 * 8), %rsp;
+	CFI_ADJUST_CFA_OFFSET(-5 * 8);
+
+	vpxord (0 * 64)(%rsi), RA0z, RA0z;
+	vpxord (1 * 64)(%rsi), RA1z, RA1z;
+	vpxord (2 * 64)(%rsi), RA2z, RA2z;
+	vpxord (3 * 64)(%rsi), RA3z, RA3z;
+	vpxord (4 * 64)(%rsi), RB0z, RB0z;
+	vpxord (5 * 64)(%rsi), RB1z, RB1z;
+	vpxord (6 * 64)(%rsi), RB2z, RB2z;
+	vpxord (7 * 64)(%rsi), RB3z, RB3z;
+
+	vmovdqu32 RA0z, (0 * 64)(%rsi);
+	vmovdqu32 RA1z, (1 * 64)(%rsi);
+	vmovdqu32 RA2z, (2 * 64)(%rsi);
+	vmovdqu32 RA3z, (3 * 64)(%rsi);
+	vmovdqu32 RB0z, (4 * 64)(%rsi);
+	vmovdqu32 RB1z, (5 * 64)(%rsi);
+	vmovdqu32 RB2z, (6 * 64)(%rsi);
+	vmovdqu32 RB3z, (7 * 64)(%rsi);
+
+	/* Checksum_i = Checksum_{i-1} xor C_i  */
+	vpternlogd $0x96, RA0z, RA1z, RA2z;
+	vpternlogd $0x96, RA3z, RB0z, RB1z;
+	vpternlogd $0x96, RB2z, RB3z, RA2z;
+	vpxord RA2z, RB1z, RTMP1z;
+
+	vextracti64x4 $1, RTMP1z, RNOT;
+	vpxor RNOT, RTMP1, RTMP1;
+	vextracti128 $1, RTMP1, RNOTx;
+	vpternlogd $0x96, (%r8), RNOTx, RTMP1x;
+	vmovdqu RTMP1x, (%r8);
+
+	vzeroall;
+
+	ret_spec_stop;
+	CFI_ENDPROC();
+ELF(.size FUNC_NAME(ocb_dec_blk32),.-FUNC_NAME(ocb_dec_blk32);)
+
+#endif /* GCRY_SM4_AVX512_AMD64_H */
diff --git a/cipher/sm4-gfni-avx512-amd64.S b/cipher/sm4-gfni-avx512-amd64.S
index 91f6e80b..8bc8bd81 100644
--- a/cipher/sm4-gfni-avx512-amd64.S
+++ b/cipher/sm4-gfni-avx512-amd64.S
@@ -120,7 +120,7 @@ SECTION_RODATA
 	.byte 0x19, 0x8b, 0x6c, 0x1e, 0x51, 0x8e, 0x2d, 0xd7
 	.byte 0x19, 0x8b, 0x6c, 0x1e, 0x51, 0x8e, 0x2d, 0xd7
 
-/* For CTR-mode IV byteswap */
+/* For output IV byteswap */
 .Lbswap128_mask:
 	.byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
 
@@ -128,53 +128,6 @@ SECTION_RODATA
 .Lbswap32_mask:
 	.byte 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12
 
-.Lcounter2222_lo:
-	.quad 2, 0
-.Lcounter4444_lo:
-	.quad 4, 0
-.Lcounter8888_lo:
-	.quad 8, 0
-.Lcounter16161616_lo:
-	.quad 16, 0
-.Lcounter1111_hi:
-	.quad 0, 1
-
-.align 64
-.Lcounter0123_lo:
-	.quad 0, 0
-	.quad 1, 0
-	.quad 2, 0
-	.quad 3, 0
-
-/* CTR byte addition constants */
-.align 64
-.Lbige_addb_0_1:
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1
-.Lbige_addb_2_3:
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3
-.Lbige_addb_4_5:
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5
-.Lbige_addb_6_7:
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7
-.Lbige_addb_8_9:
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9
-.Lbige_addb_10_11:
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11
-.Lbige_addb_12_13:
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13
-.Lbige_addb_14_15:
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15
-.Lbige_addb_16:
-	.byte 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16
-
 .text
 
 .align 16
@@ -1315,547 +1268,9 @@ __sm4_gfni_crypt_blk32:
 	CFI_ENDPROC();
 ELF(.size __sm4_gfni_crypt_blk32,.-__sm4_gfni_crypt_blk32;)
 
-.align 16
-.globl _gcry_sm4_gfni_avx512_crypt_blk32
-ELF(.type   _gcry_sm4_gfni_avx512_crypt_blk32, at function;)
-_gcry_sm4_gfni_avx512_crypt_blk32:
-	/* input:
-	 *	%rdi: ctx, CTX
-	 *	%rsi: dst (32 blocks)
-	 *	%rdx: src (32 blocks)
-	 */
-	CFI_STARTPROC();
-	spec_stop_avx512;
-
-	/* Load input */
-	vmovdqu32 (0 * 64)(%rdx), RA0z;
-	vmovdqu32 (1 * 64)(%rdx), RA1z;
-	vmovdqu32 (2 * 64)(%rdx), RA2z;
-	vmovdqu32 (3 * 64)(%rdx), RA3z;
-	vmovdqu32 (4 * 64)(%rdx), RB0z;
-	vmovdqu32 (5 * 64)(%rdx), RB1z;
-	vmovdqu32 (6 * 64)(%rdx), RB2z;
-	vmovdqu32 (7 * 64)(%rdx), RB3z;
-
-	call __sm4_gfni_crypt_blk32;
-
-	vmovdqu32 RA0z, (0 * 64)(%rsi);
-	vmovdqu32 RA1z, (1 * 64)(%rsi);
-	vmovdqu32 RA2z, (2 * 64)(%rsi);
-	vmovdqu32 RA3z, (3 * 64)(%rsi);
-	vmovdqu32 RB0z, (4 * 64)(%rsi);
-	vmovdqu32 RB1z, (5 * 64)(%rsi);
-	vmovdqu32 RB2z, (6 * 64)(%rsi);
-	vmovdqu32 RB3z, (7 * 64)(%rsi);
-
-	xorl %eax, %eax;
-	vzeroall;
-
-	ret_spec_stop;
-	CFI_ENDPROC();
-ELF(.size _gcry_sm4_gfni_avx512_crypt_blk32,.-_gcry_sm4_gfni_avx512_crypt_blk32;)
-
-.align 16
-.globl _gcry_sm4_gfni_avx512_ctr_enc_blk32
-ELF(.type   _gcry_sm4_gfni_avx512_ctr_enc_blk32, at function;)
-_gcry_sm4_gfni_avx512_ctr_enc_blk32:
-	/* input:
-	 *	%rdi: ctx, CTX
-	 *	%rsi: dst (32 blocks)
-	 *	%rdx: src (32 blocks)
-	 *	%rcx: iv (big endian, 128bit)
-	 */
-	CFI_STARTPROC();
-	spec_stop_avx512;
-
-	cmpb $(0x100 - 32), 15(%rcx);
-	jbe .Lctr_byteadd32;
-
-	vbroadcasti64x2 .Lbswap128_mask rRIP, RTMP0z;
-	vmovdqa32 .Lcounter0123_lo rRIP, RTMP1z;
-	vbroadcasti64x2 .Lcounter4444_lo rRIP, RTMP2z;
-	vbroadcasti64x2 .Lcounter8888_lo rRIP, RTMP3z;
-	vbroadcasti64x2 .Lcounter16161616_lo rRIP, RTMP4z;
-
-	/* load IV and byteswap */
-	movq 8(%rcx), %r11;
-	bswapq %r11;
-	vbroadcasti64x2 (%rcx), RB3z;
-	vpshufb RTMP0z, RB3z, RB3z;
-
-	/* check need for handling 64-bit overflow and carry */
-	cmpq $(0xffffffffffffffff - 32), %r11;
-	ja .Lhandle_ctr_carry_blk32;
-
-	/* construct IVs */
-	vpaddq RTMP1z, RB3z, RA0z; /* +0:+1:+2:+3 */
-	vpaddq RTMP2z, RA0z, RA1z; /* +4:+5:+6:+7 */
-	vpaddq RTMP3z, RA0z, RA2z; /* +8:+9:+10:+11 */
-	vpaddq RTMP3z, RA1z, RA3z; /* +12:+13:+14:+15 */
-	vpaddq RTMP4z, RA0z, RB0z; /* +16... */
-	vpaddq RTMP4z, RA1z, RB1z; /* +20... */
-	vpaddq RTMP4z, RA2z, RB2z; /* +24... */
-	vpaddq RTMP4z, RA3z, RB3z; /* +28... */
-
-	/* Update counter */
-	leaq 32(%r11), %r11;
-	bswapq %r11;
-	movq %r11, 8(%rcx);
-
-	jmp .Lctr_carry_done_blk32;
-
-.Lhandle_ctr_carry_blk32:
-	vbroadcasti64x2 .Lcounter1111_hi rRIP, RNOTz;
-
-	/* construct IVs */
-	add_le128(RA0z, RB3z, RTMP1z, RNOTz); /* +0:+1:+2:+3 */
-	add_le128(RA1z, RA0z, RTMP2z, RNOTz); /* +4:+5:+6:+7 */
-	add_le128(RA2z, RA0z, RTMP3z, RNOTz); /* +8:+9:+10:+11 */
-	add_le128(RA3z, RA1z, RTMP3z, RNOTz); /* +12:+13:+14:+15 */
-	add_le128(RB0z, RA0z, RTMP4z, RNOTz); /* +16... */
-	add_le128(RB1z, RA1z, RTMP4z, RNOTz); /* +20... */
-	add_le128(RB2z, RA2z, RTMP4z, RNOTz); /* +24... */
-	add_le128(RB3z, RA3z, RTMP4z, RNOTz); /* +28... */
-
-	/* Update counter */
-	addq $32, %r11;
-	movq (%rcx), %r10;
-	bswapq %r10;
-	adcq $0, %r10;
-	bswapq %r11;
-	bswapq %r10;
-	movq %r11, 8(%rcx);
-	movq %r10, (%rcx);
-
-.align 16
-.Lctr_carry_done_blk32:
-	/* Byte-swap IVs. */
-	vpshufb RTMP0z, RA0z, RA0z;
-	vpshufb RTMP0z, RA1z, RA1z;
-	vpshufb RTMP0z, RA2z, RA2z;
-	vpshufb RTMP0z, RA3z, RA3z;
-	vpshufb RTMP0z, RB0z, RB0z;
-	vpshufb RTMP0z, RB1z, RB1z;
-	vpshufb RTMP0z, RB2z, RB2z;
-	vpshufb RTMP0z, RB3z, RB3z;
-
-.align 16
-.Lload_ctr_done32:
-	call __sm4_gfni_crypt_blk32;
-
-	vpxord (0 * 64)(%rdx), RA0z, RA0z;
-	vpxord (1 * 64)(%rdx), RA1z, RA1z;
-	vpxord (2 * 64)(%rdx), RA2z, RA2z;
-	vpxord (3 * 64)(%rdx), RA3z, RA3z;
-	vpxord (4 * 64)(%rdx), RB0z, RB0z;
-	vpxord (5 * 64)(%rdx), RB1z, RB1z;
-	vpxord (6 * 64)(%rdx), RB2z, RB2z;
-	vpxord (7 * 64)(%rdx), RB3z, RB3z;
-
-	vmovdqu32 RA0z, (0 * 64)(%rsi);
-	vmovdqu32 RA1z, (1 * 64)(%rsi);
-	vmovdqu32 RA2z, (2 * 64)(%rsi);
-	vmovdqu32 RA3z, (3 * 64)(%rsi);
-	vmovdqu32 RB0z, (4 * 64)(%rsi);
-	vmovdqu32 RB1z, (5 * 64)(%rsi);
-	vmovdqu32 RB2z, (6 * 64)(%rsi);
-	vmovdqu32 RB3z, (7 * 64)(%rsi);
-
-	vzeroall;
-	kxorq %k1, %k1, %k1;
-
-	ret_spec_stop;
-
-.align 16
-.Lctr_byteadd_full_ctr_carry32:
-	movq 8(%rcx), %r11;
-	movq (%rcx), %r10;
-	bswapq %r11;
-	bswapq %r10;
-	addq $32, %r11;
-	adcq $0, %r10;
-	bswapq %r11;
-	bswapq %r10;
-	movq %r11, 8(%rcx);
-	movq %r10, (%rcx);
-	jmp .Lctr_byteadd_zmm32;
-.align 16
-.Lctr_byteadd32:
-	vbroadcasti64x2 (%rcx), RA3z;
-	je .Lctr_byteadd_full_ctr_carry32;
-	addb $32, 15(%rcx);
-.Lctr_byteadd_zmm32:
-	vbroadcasti64x2 .Lbige_addb_16 rRIP, RB3z;
-	vpaddb RB3z, RA3z, RB3z;
-	vpaddb .Lbige_addb_0_1 rRIP, RA3z, RA0z;
-	vpaddb .Lbige_addb_4_5 rRIP, RA3z, RA1z;
-	vpaddb .Lbige_addb_8_9 rRIP, RA3z, RA2z;
-	vpaddb .Lbige_addb_12_13 rRIP, RA3z, RA3z;
-	vpaddb .Lbige_addb_0_1 rRIP, RB3z, RB0z;
-	vpaddb .Lbige_addb_4_5 rRIP, RB3z, RB1z;
-	vpaddb .Lbige_addb_8_9 rRIP, RB3z, RB2z;
-	vpaddb .Lbige_addb_12_13 rRIP, RB3z, RB3z;
-
-	jmp .Lload_ctr_done32;
-	CFI_ENDPROC();
-ELF(.size _gcry_sm4_gfni_avx512_ctr_enc_blk32,.-_gcry_sm4_gfni_avx512_ctr_enc_blk32;)
-
-.align 16
-.globl _gcry_sm4_gfni_avx512_cbc_dec_blk32
-ELF(.type   _gcry_sm4_gfni_avx512_cbc_dec_blk32, at function;)
-_gcry_sm4_gfni_avx512_cbc_dec_blk32:
-	/* input:
-	 *	%rdi: ctx, CTX
-	 *	%rsi: dst (32 blocks)
-	 *	%rdx: src (32 blocks)
-	 *	%rcx: iv
-	 */
-	CFI_STARTPROC();
-	spec_stop_avx512;
-
-	vmovdqu32 (0 * 64)(%rdx), RA0z;
-	vmovdqu32 (1 * 64)(%rdx), RA1z;
-	vmovdqu32 (2 * 64)(%rdx), RA2z;
-	vmovdqu32 (3 * 64)(%rdx), RA3z;
-	vmovdqu32 (4 * 64)(%rdx), RB0z;
-	vmovdqu32 (5 * 64)(%rdx), RB1z;
-	vmovdqu32 (6 * 64)(%rdx), RB2z;
-	vmovdqu32 (7 * 64)(%rdx), RB3z;
-
-	call __sm4_gfni_crypt_blk32;
-
-	vmovdqu (%rcx), RNOTx;
-	vinserti64x2 $1, (0 * 16)(%rdx), RNOT, RNOT;
-	vinserti64x4 $1, (1 * 16)(%rdx), RNOTz, RNOTz;
-	vpxord RNOTz, RA0z, RA0z;
-	vpxord (0 * 64 + 48)(%rdx), RA1z, RA1z;
-	vpxord (1 * 64 + 48)(%rdx), RA2z, RA2z;
-	vpxord (2 * 64 + 48)(%rdx), RA3z, RA3z;
-	vpxord (3 * 64 + 48)(%rdx), RB0z, RB0z;
-	vpxord (4 * 64 + 48)(%rdx), RB1z, RB1z;
-	vpxord (5 * 64 + 48)(%rdx), RB2z, RB2z;
-	vpxord (6 * 64 + 48)(%rdx), RB3z, RB3z;
-	vmovdqu (7 * 64 + 48)(%rdx), RNOTx;
-	vmovdqu RNOTx, (%rcx); /* store new IV */
-
-	vmovdqu32 RA0z, (0 * 64)(%rsi);
-	vmovdqu32 RA1z, (1 * 64)(%rsi);
-	vmovdqu32 RA2z, (2 * 64)(%rsi);
-	vmovdqu32 RA3z, (3 * 64)(%rsi);
-	vmovdqu32 RB0z, (4 * 64)(%rsi);
-	vmovdqu32 RB1z, (5 * 64)(%rsi);
-	vmovdqu32 RB2z, (6 * 64)(%rsi);
-	vmovdqu32 RB3z, (7 * 64)(%rsi);
-
-	vzeroall;
-
-	ret_spec_stop;
-	CFI_ENDPROC();
-ELF(.size _gcry_sm4_gfni_avx512_cbc_dec_blk32,.-_gcry_sm4_gfni_avx512_cbc_dec_blk32;)
-
-.align 16
-.globl _gcry_sm4_gfni_avx512_cfb_dec_blk32
-ELF(.type   _gcry_sm4_gfni_avx512_cfb_dec_blk32, at function;)
-_gcry_sm4_gfni_avx512_cfb_dec_blk32:
-	/* input:
-	 *	%rdi: ctx, CTX
-	 *	%rsi: dst (32 blocks)
-	 *	%rdx: src (32 blocks)
-	 *	%rcx: iv
-	 */
-	CFI_STARTPROC();
-	spec_stop_avx512;
-
-	/* Load input */
-	vmovdqu (%rcx), RA0x;
-	vinserti64x2 $1, (%rdx), RA0, RA0;
-	vinserti64x4 $1, 16(%rdx), RA0z, RA0z;
-	vmovdqu32 (0 * 64 + 48)(%rdx), RA1z;
-	vmovdqu32 (1 * 64 + 48)(%rdx), RA2z;
-	vmovdqu32 (2 * 64 + 48)(%rdx), RA3z;
-	vmovdqu32 (3 * 64 + 48)(%rdx), RB0z;
-	vmovdqu32 (4 * 64 + 48)(%rdx), RB1z;
-	vmovdqu32 (5 * 64 + 48)(%rdx), RB2z;
-	vmovdqu32 (6 * 64 + 48)(%rdx), RB3z;
-
-	/* Update IV */
-	vmovdqu (7 * 64 + 48)(%rdx), RNOTx;
-	vmovdqu RNOTx, (%rcx);
-
-	call __sm4_gfni_crypt_blk32;
-
-	vpxord (0 * 64)(%rdx), RA0z, RA0z;
-	vpxord (1 * 64)(%rdx), RA1z, RA1z;
-	vpxord (2 * 64)(%rdx), RA2z, RA2z;
-	vpxord (3 * 64)(%rdx), RA3z, RA3z;
-	vpxord (4 * 64)(%rdx), RB0z, RB0z;
-	vpxord (5 * 64)(%rdx), RB1z, RB1z;
-	vpxord (6 * 64)(%rdx), RB2z, RB2z;
-	vpxord (7 * 64)(%rdx), RB3z, RB3z;
-
-	vmovdqu32 RA0z, (0 * 64)(%rsi);
-	vmovdqu32 RA1z, (1 * 64)(%rsi);
-	vmovdqu32 RA2z, (2 * 64)(%rsi);
-	vmovdqu32 RA3z, (3 * 64)(%rsi);
-	vmovdqu32 RB0z, (4 * 64)(%rsi);
-	vmovdqu32 RB1z, (5 * 64)(%rsi);
-	vmovdqu32 RB2z, (6 * 64)(%rsi);
-	vmovdqu32 RB3z, (7 * 64)(%rsi);
-
-	vzeroall;
-
-	ret_spec_stop;
-	CFI_ENDPROC();
-ELF(.size _gcry_sm4_gfni_avx512_cfb_dec_blk32,.-_gcry_sm4_gfni_avx512_cfb_dec_blk32;)
-
-.align 16
-.globl _gcry_sm4_gfni_avx512_ocb_enc_blk32
-ELF(.type _gcry_sm4_gfni_avx512_ocb_enc_blk32, at function;)
-_gcry_sm4_gfni_avx512_ocb_enc_blk32:
-	/* input:
-	 *	%rdi: ctx, CTX
-	 *	%rsi: dst (32 blocks)
-	 *	%rdx: src (32 blocks)
-	 *	%rcx: offset
-	 *	%r8 : checksum
-	 *	%r9 : L pointers (void *L[32])
-	 */
-	CFI_STARTPROC();
-	spec_stop_avx512;
-
-	subq $(5 * 8), %rsp;
-	CFI_ADJUST_CFA_OFFSET(5 * 8);
-
-	movq %r12, (0 * 8)(%rsp);
-	movq %r13, (1 * 8)(%rsp);
-	movq %r14, (2 * 8)(%rsp);
-	movq %r15, (3 * 8)(%rsp);
-	movq %rbx, (4 * 8)(%rsp);
-	CFI_REL_OFFSET(%r12, 0 * 8);
-	CFI_REL_OFFSET(%r13, 1 * 8);
-	CFI_REL_OFFSET(%r14, 2 * 8);
-	CFI_REL_OFFSET(%r15, 3 * 8);
-	CFI_REL_OFFSET(%rbx, 4 * 8);
-
-	vmovdqu (%rcx), RTMP0x;
-
-	/* Offset_i = Offset_{i-1} xor L_{ntz(i)} */
-	/* Checksum_i = Checksum_{i-1} xor P_i  */
-	/* C_i = Offset_i xor ENCIPHER(K, P_i xor Offset_i)  */
-
-#define OCB_INPUT(n, l0reg, l1reg, l2reg, l3reg, zreg, zplain) \
-	  vmovdqu32 (n * 64)(%rdx), zplain; \
-	  vpxor (l0reg), RTMP0x, RNOTx; \
-	  vpxor (l1reg), RNOTx, RTMP0x; \
-	  vinserti64x2 $1, RTMP0x, RNOT, RNOT; \
-	  vpxor (l2reg), RTMP0x, RTMP0x; \
-	  vinserti64x2 $2, RTMP0x, RNOTz, RNOTz; \
-	  vpxor (l3reg), RTMP0x, RTMP0x; \
-	  vinserti64x2 $3, RTMP0x, RNOTz, RNOTz; \
-	  vpxord zplain, RNOTz, zreg; \
-	  vmovdqu32 RNOTz, (n * 64)(%rsi);
-
-#define OCB_LOAD_PTRS(n) \
-	  movq ((n * 4 * 8) + (0 * 8))(%r9), %r10; \
-	  movq ((n * 4 * 8) + (1 * 8))(%r9), %r11; \
-	  movq ((n * 4 * 8) + (2 * 8))(%r9), %r12; \
-	  movq ((n * 4 * 8) + (3 * 8))(%r9), %r13; \
-	  movq ((n * 4 * 8) + (4 * 8))(%r9), %r14; \
-	  movq ((n * 4 * 8) + (5 * 8))(%r9), %r15; \
-	  movq ((n * 4 * 8) + (6 * 8))(%r9), %rax; \
-	  movq ((n * 4 * 8) + (7 * 8))(%r9), %rbx;
-
-	OCB_LOAD_PTRS(0);
-	OCB_INPUT(0, %r10, %r11, %r12, %r13, RA0z, RTMP1z);
-	OCB_INPUT(1, %r14, %r15, %rax, %rbx, RA1z, RTMP2z);
-	OCB_LOAD_PTRS(2);
-	OCB_INPUT(2, %r10, %r11, %r12, %r13, RA2z, RTMP3z);
-	vpternlogd $0x96, RTMP1z, RTMP2z, RTMP3z;
-	OCB_INPUT(3, %r14, %r15, %rax, %rbx, RA3z, RTMP4z);
-	OCB_LOAD_PTRS(4);
-	OCB_INPUT(4, %r10, %r11, %r12, %r13, RB0z, RX0z);
-	OCB_INPUT(5, %r14, %r15, %rax, %rbx, RB1z, RX1z);
-	vpternlogd $0x96, RTMP4z, RX0z, RX1z;
-	OCB_LOAD_PTRS(6);
-	OCB_INPUT(6, %r10, %r11, %r12, %r13, RB2z, RTMP4z);
-	OCB_INPUT(7, %r14, %r15, %rax, %rbx, RB3z, RX0z);
-#undef OCB_LOAD_PTRS
-#undef OCB_INPUT
-
-	vpternlogd $0x96, RTMP3z, RTMP4z, RX0z;
-	vpxord RX1z, RX0z, RNOTz;
-	vextracti64x4 $1, RNOTz, RTMP1;
-	vpxor RTMP1, RNOT, RNOT;
-	vextracti128 $1, RNOT, RTMP1x;
-	vpternlogd $0x96, (%r8), RTMP1x, RNOTx;
-
-	movq (0 * 8)(%rsp), %r12;
-	movq (1 * 8)(%rsp), %r13;
-	movq (2 * 8)(%rsp), %r14;
-	movq (3 * 8)(%rsp), %r15;
-	movq (4 * 8)(%rsp), %rbx;
-	CFI_RESTORE(%r12);
-	CFI_RESTORE(%r13);
-	CFI_RESTORE(%r14);
-	CFI_RESTORE(%r15);
-	CFI_RESTORE(%rbx);
-
-	vmovdqu RTMP0x, (%rcx);
-	vmovdqu RNOTx, (%r8);
-
-	call __sm4_gfni_crypt_blk32;
-
-	addq $(5 * 8), %rsp;
-	CFI_ADJUST_CFA_OFFSET(-5 * 8);
-
-	vpxord (0 * 64)(%rsi), RA0z, RA0z;
-	vpxord (1 * 64)(%rsi), RA1z, RA1z;
-	vpxord (2 * 64)(%rsi), RA2z, RA2z;
-	vpxord (3 * 64)(%rsi), RA3z, RA3z;
-	vpxord (4 * 64)(%rsi), RB0z, RB0z;
-	vpxord (5 * 64)(%rsi), RB1z, RB1z;
-	vpxord (6 * 64)(%rsi), RB2z, RB2z;
-	vpxord (7 * 64)(%rsi), RB3z, RB3z;
-
-	vmovdqu32 RA0z, (0 * 64)(%rsi);
-	vmovdqu32 RA1z, (1 * 64)(%rsi);
-	vmovdqu32 RA2z, (2 * 64)(%rsi);
-	vmovdqu32 RA3z, (3 * 64)(%rsi);
-	vmovdqu32 RB0z, (4 * 64)(%rsi);
-	vmovdqu32 RB1z, (5 * 64)(%rsi);
-	vmovdqu32 RB2z, (6 * 64)(%rsi);
-	vmovdqu32 RB3z, (7 * 64)(%rsi);
-
-	vzeroall;
-
-	ret_spec_stop;
-	CFI_ENDPROC();
-ELF(.size _gcry_sm4_gfni_avx512_ocb_enc_blk32,.-_gcry_sm4_gfni_avx512_ocb_enc_blk32;)
-
-.align 16
-.globl _gcry_sm4_gfni_avx512_ocb_dec_blk32
-ELF(.type _gcry_sm4_gfni_avx512_ocb_dec_blk32, at function;)
-_gcry_sm4_gfni_avx512_ocb_dec_blk32:
-	/* input:
-	 *	%rdi: ctx, CTX
-	 *	%rsi: dst (32 blocks)
-	 *	%rdx: src (32 blocks)
-	 *	%rcx: offset
-	 *	%r8 : checksum
-	 *	%r9 : L pointers (void *L[32])
-	 */
-	CFI_STARTPROC();
-	spec_stop_avx512;
-
-	subq $(5 * 8), %rsp;
-	CFI_ADJUST_CFA_OFFSET(5 * 8);
-
-	movq %r12, (0 * 8)(%rsp);
-	movq %r13, (1 * 8)(%rsp);
-	movq %r14, (2 * 8)(%rsp);
-	movq %r15, (3 * 8)(%rsp);
-	movq %rbx, (4 * 8)(%rsp);
-	CFI_REL_OFFSET(%r12, 0 * 8);
-	CFI_REL_OFFSET(%r13, 1 * 8);
-	CFI_REL_OFFSET(%r14, 2 * 8);
-	CFI_REL_OFFSET(%r15, 3 * 8);
-	CFI_REL_OFFSET(%rbx, 4 * 8);
-
-	vmovdqu (%rcx), RTMP0x;
-
-	/* Offset_i = Offset_{i-1} xor L_{ntz(i)} */
-	/* C_i = Offset_i xor DECIPHER(K, P_i xor Offset_i)  */
-
-#define OCB_INPUT(n, l0reg, l1reg, l2reg, l3reg, zreg) \
-	  vmovdqu32 (n * 64)(%rdx), RTMP1z; \
-	  vpxor (l0reg), RTMP0x, RNOTx; \
-	  vpxor (l1reg), RNOTx, RTMP0x; \
-	  vinserti64x2 $1, RTMP0x, RNOT, RNOT; \
-	  vpxor (l2reg), RTMP0x, RTMP0x; \
-	  vinserti64x2 $2, RTMP0x, RNOTz, RNOTz; \
-	  vpxor (l3reg), RTMP0x, RTMP0x; \
-	  vinserti64x2 $3, RTMP0x, RNOTz, RNOTz; \
-	  vpxord RTMP1z, RNOTz, zreg; \
-	  vmovdqu32 RNOTz, (n * 64)(%rsi);
-
-#define OCB_LOAD_PTRS(n) \
-	  movq ((n * 4 * 8) + (0 * 8))(%r9), %r10; \
-	  movq ((n * 4 * 8) + (1 * 8))(%r9), %r11; \
-	  movq ((n * 4 * 8) + (2 * 8))(%r9), %r12; \
-	  movq ((n * 4 * 8) + (3 * 8))(%r9), %r13; \
-	  movq ((n * 4 * 8) + (4 * 8))(%r9), %r14; \
-	  movq ((n * 4 * 8) + (5 * 8))(%r9), %r15; \
-	  movq ((n * 4 * 8) + (6 * 8))(%r9), %rax; \
-	  movq ((n * 4 * 8) + (7 * 8))(%r9), %rbx;
-
-	OCB_LOAD_PTRS(0);
-	OCB_INPUT(0, %r10, %r11, %r12, %r13, RA0z);
-	OCB_INPUT(1, %r14, %r15, %rax, %rbx, RA1z);
-	OCB_LOAD_PTRS(2);
-	OCB_INPUT(2, %r10, %r11, %r12, %r13, RA2z);
-	OCB_INPUT(3, %r14, %r15, %rax, %rbx, RA3z);
-	OCB_LOAD_PTRS(4);
-	OCB_INPUT(4, %r10, %r11, %r12, %r13, RB0z);
-	OCB_INPUT(5, %r14, %r15, %rax, %rbx, RB1z);
-	OCB_LOAD_PTRS(6);
-	OCB_INPUT(6, %r10, %r11, %r12, %r13, RB2z);
-	OCB_INPUT(7, %r14, %r15, %rax, %rbx, RB3z);
-#undef OCB_LOAD_PTRS
-#undef OCB_INPUT
-
-	movq (0 * 8)(%rsp), %r12;
-	movq (1 * 8)(%rsp), %r13;
-	movq (2 * 8)(%rsp), %r14;
-	movq (3 * 8)(%rsp), %r15;
-	movq (4 * 8)(%rsp), %rbx;
-	CFI_RESTORE(%r12);
-	CFI_RESTORE(%r13);
-	CFI_RESTORE(%r14);
-	CFI_RESTORE(%r15);
-	CFI_RESTORE(%rbx);
-
-	vmovdqu RTMP0x, (%rcx);
-
-	call __sm4_gfni_crypt_blk32;
-
-	addq $(5 * 8), %rsp;
-	CFI_ADJUST_CFA_OFFSET(-5 * 8);
-
-	vpxord (0 * 64)(%rsi), RA0z, RA0z;
-	vpxord (1 * 64)(%rsi), RA1z, RA1z;
-	vpxord (2 * 64)(%rsi), RA2z, RA2z;
-	vpxord (3 * 64)(%rsi), RA3z, RA3z;
-	vpxord (4 * 64)(%rsi), RB0z, RB0z;
-	vpxord (5 * 64)(%rsi), RB1z, RB1z;
-	vpxord (6 * 64)(%rsi), RB2z, RB2z;
-	vpxord (7 * 64)(%rsi), RB3z, RB3z;
-
-	vmovdqu32 RA0z, (0 * 64)(%rsi);
-	vmovdqu32 RA1z, (1 * 64)(%rsi);
-	vmovdqu32 RA2z, (2 * 64)(%rsi);
-	vmovdqu32 RA3z, (3 * 64)(%rsi);
-	vmovdqu32 RB0z, (4 * 64)(%rsi);
-	vmovdqu32 RB1z, (5 * 64)(%rsi);
-	vmovdqu32 RB2z, (6 * 64)(%rsi);
-	vmovdqu32 RB3z, (7 * 64)(%rsi);
-
-	/* Checksum_i = Checksum_{i-1} xor C_i  */
-	vpternlogd $0x96, RA0z, RA1z, RA2z;
-	vpternlogd $0x96, RA3z, RB0z, RB1z;
-	vpternlogd $0x96, RB2z, RB3z, RA2z;
-	vpxord RA2z, RB1z, RTMP1z;
-
-	vextracti64x4 $1, RTMP1z, RNOT;
-	vpxor RNOT, RTMP1, RTMP1;
-	vextracti128 $1, RTMP1, RNOTx;
-	vpternlogd $0x96, (%r8), RNOTx, RTMP1x;
-	vmovdqu RTMP1x, (%r8);
-
-	vzeroall;
-
-	ret_spec_stop;
-	CFI_ENDPROC();
-ELF(.size _gcry_sm4_gfni_avx512_ocb_dec_blk32,.-_gcry_sm4_gfni_avx512_ocb_dec_blk32;)
+#define FUNC_NAME(func) _gcry_sm4_gfni_avx512_ ## func
+#define SM4_CRYPT_BLK32 __sm4_gfni_crypt_blk32
+#include "sm4-avx512-amd64.h"
 
 #endif /*defined(ENABLE_GFNI_SUPPORT) && defined(ENABLE_AVX512_SUPPORT)*/
 #endif /*__x86_64*/
diff --git a/cipher/sm4-intel-avx512-amd64.S b/cipher/sm4-intel-avx512-amd64.S
new file mode 100644
index 00000000..a4963db0
--- /dev/null
+++ b/cipher/sm4-intel-avx512-amd64.S
@@ -0,0 +1,174 @@
+/* sm4-intel-avx512-amd64.S  -  Intel SM4 instructions AVX512 SM4 cipher
+ *
+ * Copyright (C) 2026 Jussi Kivilinna <jussi.kivilinna at iki.fi>
+ *
+ * This file is part of Libgcrypt.
+ *
+ * Libgcrypt is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * Libgcrypt is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <config.h>
+
+#ifdef __x86_64
+#if (defined(HAVE_COMPATIBLE_GCC_AMD64_PLATFORM_AS) || \
+     defined(HAVE_COMPATIBLE_GCC_WIN64_PLATFORM_AS)) && \
+    defined(ENABLE_AVX512_SUPPORT) && defined(HAVE_GCC_INLINE_ASM_SM4)
+
+#include "asm-common-amd64.h"
+
+/* vector registers */
+#define RA0          %ymm0
+#define RA1          %ymm1
+#define RA2          %ymm2
+#define RA3          %ymm3
+#define RA0x         %xmm0
+#define RA1x         %xmm1
+#define RA2x         %xmm2
+#define RA3x         %xmm3
+#define RA0z         %zmm0
+#define RA1z         %zmm1
+#define RA2z         %zmm2
+#define RA3z         %zmm3
+
+#define RTMP0        %ymm4
+#define RTMP1        %ymm5
+#define RTMP2        %ymm6
+#define RTMP3        %ymm7
+#define RTMP4        %ymm8
+#define RTMP0x       %xmm4
+#define RTMP1x       %xmm5
+#define RTMP2x       %xmm6
+#define RTMP3x       %xmm7
+#define RTMP4x       %xmm8
+#define RTMP0z       %zmm4
+#define RTMP1z       %zmm5
+#define RTMP2z       %zmm6
+#define RTMP3z       %zmm7
+#define RTMP4z       %zmm8
+
+#define RB0          %ymm9
+#define RB1          %ymm10
+#define RB2          %ymm11
+#define RB3          %ymm12
+#define RB0x         %xmm9
+#define RB1x         %xmm10
+#define RB2x         %xmm11
+#define RB3x         %xmm12
+#define RB0z         %zmm9
+#define RB1z         %zmm10
+#define RB2z         %zmm11
+#define RB3z         %zmm12
+
+#define RX0          %ymm13
+#define RX1          %ymm14
+#define RX0x         %xmm13
+#define RX1x         %xmm14
+#define RX0z         %zmm13
+#define RX1z         %zmm14
+
+#define RNOT         %ymm15
+#define RNOTx        %xmm15
+#define RNOTz        %zmm15
+
+SECTION_RODATA
+.align 32
+
+ELF(.type _sm4_intel_avx512_consts, at object)
+_sm4_intel_avx512_consts:
+
+/* For output word byte-swap */
+.Lbswap128_mask:
+	.byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
+
+/* For input word byte-swap */
+.Lbswap32_mask:
+	.byte 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12
+
+.text
+
+/**********************************************************************
+  32-way SM4 with Intel SM4 instructions
+ **********************************************************************/
+
+.align 16
+ELF(.type   __sm4_intel_crypt_blk32, at function;)
+__sm4_intel_crypt_blk32:
+	/* input:
+	 *	%rdi: round key array, CTX
+	 *	RA0z, RA1z, RA2z, RA3z, RB0z, RB1z, RB2z, RB3z: 32 parallel
+	 *						plaintext blocks
+	 * output:
+	 *	RA0z, RA1z, RA2z, RA3z, RB0z, RB1z, RB2z, RB3z: 32 parallel
+	 *						ciphertext blocks
+	 */
+	CFI_STARTPROC();
+
+	vbroadcasti32x4 .Lbswap32_mask rRIP, RTMP0z;
+	vpshufb RTMP0z, RA0z, RA0z;
+	vpshufb RTMP0z, RA1z, RA1z;
+	vpshufb RTMP0z, RA2z, RA2z;
+	vpshufb RTMP0z, RA3z, RA3z;
+	vpshufb RTMP0z, RB0z, RB0z;
+	vpshufb RTMP0z, RB1z, RB1z;
+	vpshufb RTMP0z, RB2z, RB2z;
+	vpshufb RTMP0z, RB3z, RB3z;
+
+#define ROUND4(rkoff) \
+	vbroadcasti32x4 (rkoff)(%rdi), RTMP1z; \
+	vsm4rnds4 RTMP1z, RA0z, RA0z; \
+	vsm4rnds4 RTMP1z, RA1z, RA1z; \
+	vsm4rnds4 RTMP1z, RA2z, RA2z; \
+	vsm4rnds4 RTMP1z, RA3z, RA3z; \
+	vsm4rnds4 RTMP1z, RB0z, RB0z; \
+	vsm4rnds4 RTMP1z, RB1z, RB1z; \
+	vsm4rnds4 RTMP1z, RB2z, RB2z; \
+	vsm4rnds4 RTMP1z, RB3z, RB3z;
+
+	ROUND4(0 * 16);
+	ROUND4(1 * 16);
+	ROUND4(2 * 16);
+	ROUND4(3 * 16);
+	ROUND4(4 * 16);
+	ROUND4(5 * 16);
+	ROUND4(6 * 16);
+	ROUND4(7 * 16);
+
+#undef ROUND4
+
+	vbroadcasti32x4 .Lbswap128_mask rRIP, RTMP0z;
+	vpshufb RTMP0z, RA0z, RA0z;
+	vpshufb RTMP0z, RA1z, RA1z;
+	vpshufb RTMP0z, RA2z, RA2z;
+	vpshufb RTMP0z, RA3z, RA3z;
+	vpshufb RTMP0z, RB0z, RB0z;
+	vpshufb RTMP0z, RB1z, RB1z;
+	vpshufb RTMP0z, RB2z, RB2z;
+	vpshufb RTMP0z, RB3z, RB3z;
+
+	ret_spec_stop;
+	CFI_ENDPROC();
+ELF(.size __sm4_intel_crypt_blk32,.-__sm4_intel_crypt_blk32;)
+
+#define add_le128(out, in, lo_counter, hi_counter1) \
+	vpaddq lo_counter, in, out; \
+	vpcmpuq $1, lo_counter, out, %k1; \
+	kaddb %k1, %k1, %k1; \
+	vpaddq hi_counter1, out, out{%k1};
+
+#define FUNC_NAME(func) _gcry_sm4_intel_avx512_ ## func
+#define SM4_CRYPT_BLK32 __sm4_intel_crypt_blk32
+#include "sm4-avx512-amd64.h"
+
+#endif /*defined(ENABLE_AVX512_SUPPORT) && defined(HAVE_GCC_INLINE_ASM_SM4)*/
+#endif /*__x86_64*/
diff --git a/cipher/sm4.c b/cipher/sm4.c
index e397e452..37e1a6c3 100644
--- a/cipher/sm4.c
+++ b/cipher/sm4.c
@@ -84,12 +84,22 @@
 # endif
 #endif
 
+/* USE_INTEL_SM4_AVX512 indicates whether to compile with Intel SM4
+ * instructions (VSM4RNDS4) based AVX512 code. */
+#undef USE_INTEL_SM4_AVX512
+#if defined(ENABLE_AVX512_SUPPORT) && defined(HAVE_GCC_INLINE_ASM_SM4)
+# if defined(__x86_64__) && (defined(HAVE_COMPATIBLE_GCC_AMD64_PLATFORM_AS) || \
+     defined(HAVE_COMPATIBLE_GCC_WIN64_PLATFORM_AS))
+#  define USE_INTEL_SM4_AVX512 1
+# endif
+#endif
+
 /* Assembly implementations use SystemV ABI, ABI conversion and additional
  * stack to store XMM6-XMM15 needed on Win64. */
 #undef ASM_FUNC_ABI
 #if defined(USE_AESNI_AVX) || defined(USE_AESNI_AVX2) || \
     defined(USE_GFNI_AVX2) || defined(USE_GFNI_AVX512) || \
-    defined(USE_INTEL_SM4_AVX2)
+    defined(USE_INTEL_SM4_AVX2) || defined(USE_INTEL_SM4_AVX512)
 # ifdef HAVE_COMPATIBLE_GCC_WIN64_PLATFORM_AS
 #  define ASM_FUNC_ABI __attribute__((sysv_abi))
 # else
@@ -182,6 +192,9 @@ typedef struct
 #ifdef USE_INTEL_SM4_AVX2
   unsigned int use_intel_sm4_avx2:1;
 #endif
+#ifdef USE_INTEL_SM4_AVX512
+  unsigned int use_intel_sm4_avx512:1;
+#endif
 #ifdef USE_AARCH64_SIMD
   unsigned int use_aarch64_simd:1;
 #endif
@@ -419,6 +432,38 @@ sm4_intel_avx2_crypt_blk1_16(void *rk, byte *out, const byte *in,
 }
 #endif /* USE_INTEL_SM4_AVX2 */
 
+#ifdef USE_INTEL_SM4_AVX512
+extern void _gcry_sm4_intel_avx512_ctr_enc_blk32(const u32 *rk_enc, byte *out,
+						 const byte *in,
+						 byte *ctr) ASM_FUNC_ABI;
+
+extern void _gcry_sm4_intel_avx512_cbc_dec_blk32(const u32 *rk_dec, byte *out,
+						 const byte *in,
+						 byte *iv) ASM_FUNC_ABI;
+
+extern void _gcry_sm4_intel_avx512_cfb_dec_blk32(const u32 *rk_enc, byte *out,
+						 const byte *in,
+						 byte *iv) ASM_FUNC_ABI;
+
+extern void _gcry_sm4_intel_avx512_ocb_enc_blk32(const u32 *rk_enc,
+						 unsigned char *out,
+						 const unsigned char *in,
+						 unsigned char *offset,
+						 unsigned char *checksum,
+						 const u64 Ls[32]) ASM_FUNC_ABI;
+
+extern void _gcry_sm4_intel_avx512_ocb_dec_blk32(const u32 *rk_dec,
+						 unsigned char *out,
+						 const unsigned char *in,
+						 unsigned char *offset,
+						 unsigned char *checksum,
+						 const u64 Ls[32]) ASM_FUNC_ABI;
+
+extern unsigned int
+_gcry_sm4_intel_avx512_crypt_blk32(u32 *rk, byte *out,
+				   const byte *in) ASM_FUNC_ABI;
+#endif /* USE_INTEL_SM4_AVX512 */
+
 #ifdef USE_GFNI_AVX2
 extern void _gcry_sm4_gfni_avx2_expand_key(const byte *key, u32 *rk_enc,
                                            u32 *rk_dec, const u32 *fk,
@@ -873,6 +918,9 @@ sm4_setkey (void *context, const byte *key, const unsigned keylen,
 #ifdef USE_INTEL_SM4_AVX2
   ctx->use_intel_sm4_avx2 = (hwf & HWF_INTEL_SM4) && (hwf & HWF_INTEL_AVX2);
 #endif
+#ifdef USE_INTEL_SM4_AVX512
+  ctx->use_intel_sm4_avx512 = (hwf & HWF_INTEL_SM4) && (hwf & HWF_INTEL_AVX512);
+#endif
 #ifdef USE_AARCH64_SIMD
   ctx->use_aarch64_simd = !!(hwf & HWF_ARM_NEON);
 #endif
@@ -1188,6 +1236,21 @@ _gcry_sm4_ctr_enc(void *context, unsigned char *ctr,
   const byte *inbuf = inbuf_arg;
   int burn_stack_depth = 0;
 
+#ifdef USE_INTEL_SM4_AVX512
+  if (ctx->use_intel_sm4_avx512)
+    {
+      /* Process data in 32 block chunks. */
+      while (nblocks >= 32)
+        {
+          _gcry_sm4_intel_avx512_ctr_enc_blk32(ctx->rkey_enc, outbuf, inbuf, ctr);
+
+          nblocks -= 32;
+          outbuf += 32 * 16;
+          inbuf += 32 * 16;
+        }
+    }
+#endif
+
 #ifdef USE_INTEL_SM4_AVX2
   if (ctx->use_intel_sm4_avx2)
     {
@@ -1353,6 +1416,21 @@ _gcry_sm4_cbc_dec(void *context, unsigned char *iv,
   const unsigned char *inbuf = inbuf_arg;
   int burn_stack_depth = 0;
 
+#ifdef USE_INTEL_SM4_AVX512
+  if (ctx->use_intel_sm4_avx512)
+    {
+      /* Process data in 32 block chunks. */
+      while (nblocks >= 32)
+        {
+          _gcry_sm4_intel_avx512_cbc_dec_blk32(ctx->rkey_dec, outbuf, inbuf, iv);
+
+          nblocks -= 32;
+          outbuf += 32 * 16;
+          inbuf += 32 * 16;
+        }
+    }
+#endif
+
 #ifdef USE_INTEL_SM4_AVX2
   if (ctx->use_intel_sm4_avx2)
     {
@@ -1517,6 +1595,21 @@ _gcry_sm4_cfb_dec(void *context, unsigned char *iv,
   const unsigned char *inbuf = inbuf_arg;
   int burn_stack_depth = 0;
 
+#ifdef USE_INTEL_SM4_AVX512
+  if (ctx->use_intel_sm4_avx512)
+    {
+      /* Process data in 32 block chunks. */
+      while (nblocks >= 32)
+        {
+          _gcry_sm4_intel_avx512_cfb_dec_blk32(ctx->rkey_enc, outbuf, inbuf, iv);
+
+          nblocks -= 32;
+          outbuf += 32 * 16;
+          inbuf += 32 * 16;
+        }
+    }
+#endif
+
 #ifdef USE_INTEL_SM4_AVX2
   if (ctx->use_intel_sm4_avx2)
     {
@@ -1679,6 +1772,12 @@ sm4_crypt_blk1_32 (SM4_context *ctx, byte *outbuf, const byte *inbuf,
 
   gcry_assert (num_blks <= 32);
 
+#ifdef USE_INTEL_SM4_AVX512
+  if (num_blks == 32 && ctx->use_intel_sm4_avx512)
+    {
+      return _gcry_sm4_intel_avx512_crypt_blk32 (rk, outbuf, inbuf);
+    }
+#endif
 #ifdef USE_GFNI_AVX512
   if (num_blks == 32 && ctx->use_gfni_avx512)
     {
@@ -1837,6 +1936,39 @@ _gcry_sm4_ocb_crypt (gcry_cipher_hd_t c, void *outbuf_arg,
   u64 blkn = c->u_mode.ocb.data_nblocks;
   int burn_stack_depth = 0;
 
+#ifdef USE_INTEL_SM4_AVX512
+  if (ctx->use_intel_sm4_avx512)
+    {
+      u64 Ls[32];
+      u64 *l;
+
+      if (nblocks >= 32)
+	{
+          l = bulk_ocb_prepare_L_pointers_array_blk32 (c, Ls, blkn);
+
+	  /* Process data in 32 block chunks. */
+	  while (nblocks >= 32)
+	    {
+	      blkn += 32;
+	      *l = (uintptr_t)(void *)ocb_get_l (c, blkn - blkn % 32);
+
+	      if (encrypt)
+		_gcry_sm4_intel_avx512_ocb_enc_blk32 (ctx->rkey_enc, outbuf,
+                                                      inbuf, c->u_iv.iv,
+                                                      c->u_ctr.ctr, Ls);
+	      else
+		_gcry_sm4_intel_avx512_ocb_dec_blk32 (ctx->rkey_dec, outbuf,
+                                                      inbuf, c->u_iv.iv,
+                                                      c->u_ctr.ctr, Ls);
+
+	      nblocks -= 32;
+	      outbuf += 32 * 16;
+	      inbuf += 32 * 16;
+	    }
+	}
+    }
+#endif
+
 #ifdef USE_INTEL_SM4_AVX2
   if (ctx->use_intel_sm4_avx2)
     {
diff --git a/configure.ac b/configure.ac
index f12b590e..604f063e 100644
--- a/configure.ac
+++ b/configure.ac
@@ -3874,6 +3874,7 @@ if test "$found" = "1" ; then
          GCRYPT_ASM_CIPHERS="$GCRYPT_ASM_CIPHERS sm4-gfni-avx2-amd64.lo"
          GCRYPT_ASM_CIPHERS="$GCRYPT_ASM_CIPHERS sm4-gfni-avx512-amd64.lo"
          GCRYPT_ASM_CIPHERS="$GCRYPT_ASM_CIPHERS sm4-intel-avx2-amd64.lo"
+         GCRYPT_ASM_CIPHERS="$GCRYPT_ASM_CIPHERS sm4-intel-avx512-amd64.lo"
       ;;
       aarch64-*-*)
          # Build with the assembly implementation
-- 
2.53.0