[PATCH 07/10] sm4: reduce CTR bulk counter carry handling to 16 bits
Jussi Kivilinna
jussi.kivilinna at iki.fi
Fri Jul 24 20:50:14 CEST 2026
* cipher/sm4.c (sm4_setkey): Assign ctr16be_enc bulk op.
* cipher/sm4-aesni-avx-amd64.S (inc_le128): Remove.
(_gcry_sm4_aesni_avx_ctr_enc): Add to low 16 counter bits only, drop
full-width carry handling.
* cipher/sm4-avx2-amd64.h: Likewise.
* cipher/sm4-avx512-amd64.h (ctr_enc_blk32): Add to low 16 counter bits
only, drop full-width carry handling.
* cipher/sm4-intel-avx512-amd64.S (add_le128): Remove.
* cipher/sm4-gfni-avx512-amd64.S (add_le128): Remove.
(_gcry_sm4_gfni_avx512_ctr_enc): Add to low 16 counter bits only, drop
full-width carry handling.
* cipher/sm4-aarch64.S: Likewise.
* cipher/sm4-armv8-aarch64-ce.S: Likewise.
* cipher/sm4-armv9-aarch64-sve-ce.S: Likewise.
--
SM4 CTR implementations did full 128-bit counter increment with
separate 64-bit overflow path. Under ctr16be_enc contract generic ctr
code splits work at low 16-bit overflow, so bulk function adds only to
low 16 counter bits. Drop overflow paths and use 16-bit stepping.
Moves sm4 off transitional ctr_enc alias.
Signed-off-by: Jussi Kivilinna <jussi.kivilinna at iki.fi>
---
cipher/sm4-aarch64.S | 40 +++++++-------
cipher/sm4-aesni-avx-amd64.S | 49 +++++------------
cipher/sm4-armv8-aarch64-ce.S | 39 +++++++-------
cipher/sm4-armv9-aarch64-sve-ce.S | 88 ++++++++++++++-----------------
cipher/sm4-avx2-amd64.h | 85 +++++------------------------
cipher/sm4-avx512-amd64.h | 77 +++++----------------------
cipher/sm4-gfni-avx512-amd64.S | 81 +++++-----------------------
cipher/sm4-intel-avx512-amd64.S | 6 ---
cipher/sm4.c | 2 +-
9 files changed, 132 insertions(+), 335 deletions(-)
diff --git a/cipher/sm4-aarch64.S b/cipher/sm4-aarch64.S
index c71bf3ca..0e754a88 100644
--- a/cipher/sm4-aarch64.S
+++ b/cipher/sm4-aarch64.S
@@ -595,30 +595,31 @@ _gcry_sm4_aarch64_ctr_enc:
preload_sbox(x5);
- ldp x7, x8, [x3];
- rev x7, x7;
- rev x8, x8;
+ ld1 {RIV.16b}, [x3];
+ rev16 RIV.16b, RIV.16b;
+
+ mov w7, #1;
.Lctr_loop_blk:
subs x4, x4, #8;
bmi .Lctr_end;
-#define inc_le128(vctr) \
- mov vctr.d[1], x8; \
- mov vctr.d[0], x7; \
- adds x8, x8, #1; \
- adc x7, x7, xzr; \
- rev64 vctr.16b, vctr.16b;
+#define inc_be16(vctr) \
+ rev16 vctr.16b, RIV.16b; \
+ add RIV.8h, RIV.8h, RTMP0.8h;
+
+ movi RTMP0.16b, #0;
+ mov RTMP0.b[14], w7;
/* construct CTRs */
- inc_le128(v0); /* +0 */
- inc_le128(v1); /* +1 */
- inc_le128(v2); /* +2 */
- inc_le128(v3); /* +3 */
- inc_le128(v4); /* +4 */
- inc_le128(v5); /* +5 */
- inc_le128(v6); /* +6 */
- inc_le128(v7); /* +7 */
+ inc_be16(v0); /* +0 */
+ inc_be16(v1); /* +1 */
+ inc_be16(v2); /* +2 */
+ inc_be16(v3); /* +3 */
+ inc_be16(v4); /* +4 */
+ inc_be16(v5); /* +5 */
+ inc_be16(v6); /* +6 */
+ inc_be16(v7); /* +7 */
bl __sm4_crypt_blk8;
@@ -640,9 +641,8 @@ _gcry_sm4_aarch64_ctr_enc:
.Lctr_end:
/* store new CTR */
- rev x7, x7;
- rev x8, x8;
- stp x7, x8, [x3];
+ rev16 RIV.16b, RIV.16b;
+ st1 {RIV.16b}, [x3];
VPOP_ABI;
clear_volatile_vec_regs();
diff --git a/cipher/sm4-aesni-avx-amd64.S b/cipher/sm4-aesni-avx-amd64.S
index ca9be44a..643ab276 100644
--- a/cipher/sm4-aesni-avx-amd64.S
+++ b/cipher/sm4-aesni-avx-amd64.S
@@ -571,37 +571,30 @@ _gcry_sm4_aesni_avx_ctr_enc:
vpshufb RBSWAP, RA0, RTMP0; /* be => le */
vpcmpeqd RNOT, RNOT, RNOT;
- vpsrldq $8, RNOT, RNOT; /* low: -1, high: 0 */
-
-#define inc_le128(x, minus_one, tmp) \
- vpcmpeqq minus_one, x, tmp; \
- vpsubq minus_one, x, x; \
- vpslldq $8, tmp, tmp; \
- vpsubq tmp, x, x;
+ vpsrldq $14, RNOT, RNOT; /* low: -1, high: 0 */
/* construct IVs */
- inc_le128(RTMP0, RNOT, RTMP2); /* +1 */
+ vpsubw RNOT, RTMP0, RTMP0; /* +1 */
vpshufb RBSWAP, RTMP0, RA1;
- inc_le128(RTMP0, RNOT, RTMP2); /* +2 */
+ vpsubw RNOT, RTMP0, RTMP0; /* +2 */
vpshufb RBSWAP, RTMP0, RA2;
- inc_le128(RTMP0, RNOT, RTMP2); /* +3 */
+ vpsubw RNOT, RTMP0, RTMP0; /* +3 */
vpshufb RBSWAP, RTMP0, RA3;
- inc_le128(RTMP0, RNOT, RTMP2); /* +4 */
+ vpsubw RNOT, RTMP0, RTMP0; /* +4 */
vpshufb RBSWAP, RTMP0, RB0;
- inc_le128(RTMP0, RNOT, RTMP2); /* +5 */
+ vpsubw RNOT, RTMP0, RTMP0; /* +5 */
vpshufb RBSWAP, RTMP0, RB1;
- inc_le128(RTMP0, RNOT, RTMP2); /* +6 */
+ vpsubw RNOT, RTMP0, RTMP0; /* +6 */
vpshufb RBSWAP, RTMP0, RB2;
- inc_le128(RTMP0, RNOT, RTMP2); /* +7 */
+ vpsubw RNOT, RTMP0, RTMP0; /* +7 */
vpshufb RBSWAP, RTMP0, RB3;
- inc_le128(RTMP0, RNOT, RTMP2); /* +8 */
- vpshufb RBSWAP, RTMP0, RTMP1;
-
- /* store new IV */
- vmovdqu RTMP1, (%rcx);
.align 8
.Lload_ctr_done:
+ /* Update counter */
+ addb $8, 15(%rcx);
+ adcb $0, 14(%rcx);
+
call __sm4_crypt_blk8;
vpxor (0 * 16)(%rdx), RA0, RA0;
@@ -625,26 +618,10 @@ _gcry_sm4_aesni_avx_ctr_enc:
vzeroall;
ret_spec_stop;
- .align 8
-
-.Lctr_byteadd_full_ctr_carry:
- movq 8(%rcx), %r11;
- movq (%rcx), %r10;
- bswapq %r11;
- bswapq %r10;
- addq $8, %r11;
- adcq $0, %r10;
- bswapq %r11;
- bswapq %r10;
- movq %r11, 8(%rcx);
- movq %r10, (%rcx);
- jmp .Lctr_byteadd_xmm;
+
.align 8
.Lctr_byteadd:
vmovdqu (%rcx), RA0;
- je .Lctr_byteadd_full_ctr_carry;
- addb $8, 15(%rcx);
-.Lctr_byteadd_xmm:
vpaddb .Lbige_addb_1 rRIP, RA0, RA1;
vpaddb .Lbige_addb_2 rRIP, RA0, RA2;
vpaddb .Lbige_addb_3 rRIP, RA0, RA3;
diff --git a/cipher/sm4-armv8-aarch64-ce.S b/cipher/sm4-armv8-aarch64-ce.S
index 5917ad5b..cd4a505f 100644
--- a/cipher/sm4-armv8-aarch64-ce.S
+++ b/cipher/sm4-armv8-aarch64-ce.S
@@ -546,30 +546,30 @@ _gcry_sm4_armv8_ce_ctr_enc:
load_rkey(x0);
- ldp x7, x8, [x3];
- rev x7, x7;
- rev x8, x8;
+ ld1 {RIV.16b}, [x3];
+ rev16 RIV.16b, RIV.16b;
+
+ mov w7, #1;
+ movi RMASK.16b, #0;
+ mov RMASK.b[14], w7;
.Lctr_loop_blk:
subs x4, x4, #8;
bmi .Lctr_end;
-#define inc_le128(vctr) \
- mov vctr.d[1], x8; \
- mov vctr.d[0], x7; \
- adds x8, x8, #1; \
- adc x7, x7, xzr; \
- rev64 vctr.16b, vctr.16b;
+#define inc_be16(vctr) \
+ rev16 vctr.16b, RIV.16b; \
+ add RIV.8h, RIV.8h, RMASK.8h;
/* construct CTRs */
- inc_le128(v0); /* +0 */
- inc_le128(v1); /* +1 */
- inc_le128(v2); /* +2 */
- inc_le128(v3); /* +3 */
- inc_le128(v4); /* +4 */
- inc_le128(v5); /* +5 */
- inc_le128(v6); /* +6 */
- inc_le128(v7); /* +7 */
+ inc_be16(v0); /* +0 */
+ inc_be16(v1); /* +1 */
+ inc_be16(v2); /* +2 */
+ inc_be16(v3); /* +3 */
+ inc_be16(v4); /* +4 */
+ inc_be16(v5); /* +5 */
+ inc_be16(v6); /* +6 */
+ inc_be16(v7); /* +7 */
crypt_blk8(v0, v1, v2, v3, v4, v5, v6, v7);
@@ -591,9 +591,8 @@ _gcry_sm4_armv8_ce_ctr_enc:
.Lctr_end:
/* store new CTR */
- rev x7, x7;
- rev x8, x8;
- stp x7, x8, [x3];
+ rev16 RIV.16b, RIV.16b;
+ st1 {RIV.16b}, [x3];
CLEAR_ALL_REGS();
ret_spec_stop;
diff --git a/cipher/sm4-armv9-aarch64-sve-ce.S b/cipher/sm4-armv9-aarch64-sve-ce.S
index ba9bde84..ca6e51a3 100644
--- a/cipher/sm4-armv9-aarch64-sve-ce.S
+++ b/cipher/sm4-armv9-aarch64-sve-ce.S
@@ -70,7 +70,8 @@ _gcry_sm4_armv9_svesm4_consts:
.byte 0xfc, 0xfd, 0xfe, 0xff, 0xf8, 0xf9, 0xfa, 0xfb
.byte 0xf4, 0xf5, 0xf6, 0xf7, 0xf0, 0xf1, 0xf2, 0xf3
-.Lle128_inc:
+.Lbe16_inc:
+ .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
.byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
.byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
.byte 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
@@ -102,18 +103,15 @@ _gcry_sm4_armv9_svesm4_consts:
.byte 0x0e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
.byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
.byte 0x0f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
- .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
ELF(.size _gcry_sm4_armv9_svesm4_consts,.-_gcry_sm4_armv9_svesm4_consts)
/* Register macros */
-#define RCTR z16
-#define RCTRv v16
#define RIV z16
#define RIVv v16
#define RSWAP128 z17
-#define RZERO z18
-#define RLE128_INC z19
+#define RBE16_INC z18
+#define RADDv v19
#define RTMP0 z20
#define RTMP1 z21
@@ -814,38 +812,35 @@ _gcry_sm4_armv9_sve_ce_ctr_enc:
PREPARE();
- dup RZERO.d, #0;
- GET_DATA_POINTER(x6, .Lle128_inc);
- ld1b {RLE128_INC.b}, p0/z, [x6];
-
- ldp x7, x8, [x3];
- rev x7, x7;
- rev x8, x8;
-
-#define inc_le128(zctr) \
- mov RCTRv.d[1], x8; \
- mov RCTRv.d[0], x7; \
- mov zctr.d, RLE128_INC.d; \
- dup RCTR.q, RCTR.q[0]; \
- adds x8, x8, x5, LSR #4; \
- adc x7, x7, xzr; \
- adclt zctr.d, RCTR.d, RZERO.d; \
- adclt RCTR.d, zctr.d, RZERO.d; \
- trn1 zctr.d, RCTR.d, zctr.d; \
+ GET_DATA_POINTER(x6, .Lbe16_inc);
+ ld1b {RBE16_INC.b}, p0/z, [x6];
+
+ ld1 {RIVv.16b}, [x3];
+ rev64 RIVv.16b, RIVv.16b;
+
+ lsr x6, x5, #4;
+ movi RADDv.16b, #0;
+ mov RADDv.d[1], x6;
+ mov x6, #1;
+
+#define inc_be16(zctr) \
+ dup zctr.q, RIV.q[0]; \
+ add RIVv.8h, RIVv.8h, RADDv.8h; \
+ add zctr.h, zctr.h, RBE16_INC.h; \
revb zctr.d, p0/m, zctr.d;
.Lctr_loop_blks:
sub x4, x4, x5, LSR #1; /* x4 - (8 * VL) */
tbnz x4, #63, .Lctr_tail8;
- inc_le128(z0);
- inc_le128(z1);
- inc_le128(z2);
- inc_le128(z3);
- inc_le128(z4);
- inc_le128(z5);
- inc_le128(z6);
- inc_le128(z7);
+ inc_be16(z0);
+ inc_be16(z1);
+ inc_be16(z2);
+ inc_be16(z3);
+ inc_be16(z4);
+ inc_be16(z5);
+ inc_be16(z6);
+ inc_be16(z7);
SM4_SVE_CE_CRYPT_BLK8(z0, z1, z2, z3, z4, z5, z6, z7);
@@ -887,10 +882,10 @@ _gcry_sm4_armv9_sve_ce_ctr_enc:
sub x4, x4, x5, LSR #2; /* x4 - (4 * VL) */
- inc_le128(z0);
- inc_le128(z1);
- inc_le128(z2);
- inc_le128(z3);
+ inc_be16(z0);
+ inc_be16(z1);
+ inc_be16(z2);
+ inc_be16(z3);
SM4_SVE_CE_CRYPT_BLK4(z0, z1, z2, z3);
@@ -917,7 +912,7 @@ _gcry_sm4_armv9_sve_ce_ctr_enc:
sub x4, x4, x5, LSR #4; /* x4 - VL */
- inc_le128(z0);
+ inc_be16(z0);
SM4_SVE_CE_CRYPT_BLK(z0);
ld1b {RTMP0.b}, p0/z, [x2];
eor z0.d, z0.d, RTMP0.d;
@@ -929,27 +924,26 @@ _gcry_sm4_armv9_sve_ce_ctr_enc:
b .Lctr_tail4;
.Lctr_tail:
+ mov RADDv.d[1], x6;
+
+.Lctr_tail_loop:
sub x4, x4, #1;
- /* inc_le128 for CE */
- mov v0.d[1], x8;
- mov v0.d[0], x7;
- adds x8, x8, #1;
- adc x7, x7, xzr;
- rev64 v0.16b, v0.16b;
+ /* increment counter for CE */
+ rev64 v0.16b, RIVv.16b;
+ add RIVv.8h, RIVv.8h, RADDv.8h;
SM4_CE_CRYPT_BLK(v0);
ld1 {RTMP0v.16b}, [x2], #16;
eor v0.16b, v0.16b, RTMP0v.16b;
st1 {v0.16b}, [x1], #16;
- cbnz x4, .Lctr_tail;
+ cbnz x4, .Lctr_tail_loop;
.Lctr_end:
/* store new CTR */
- rev x7, x7;
- rev x8, x8;
- stp x7, x8, [x3];
+ rev64 RIVv.16b, RIVv.16b;
+ st1 {RIVv.16b}, [x3];
CLEAR_ALL_REGS();
ret_spec_stop;
diff --git a/cipher/sm4-avx2-amd64.h b/cipher/sm4-avx2-amd64.h
index 6d3740dd..a1b68833 100644
--- a/cipher/sm4-avx2-amd64.h
+++ b/cipher/sm4-avx2-amd64.h
@@ -56,12 +56,6 @@ FUNC_NAME(cipher_mode_consts):
.text
-#define inc_le128(x, minus_one, tmp) \
- vpcmpeqq minus_one, x, tmp; \
- vpsubq minus_one, x, x; \
- vpslldq $8, tmp, tmp; \
- vpsubq tmp, x, x;
-
.align 16
.globl FUNC_NAME(ctr_enc)
ELF(.type FUNC_NAME(ctr_enc), at function;)
@@ -82,74 +76,39 @@ FUNC_NAME(ctr_enc):
vbroadcasti128 .Lbswap128_mask rRIP, RTMP3;
vpcmpeqd RNOT, RNOT, RNOT;
- vpsrldq $8, RNOT, RNOT; /* ab: -1:0 ; cd: -1:0 */
- vpaddq RNOT, RNOT, RTMP2; /* ab: -2:0 ; cd: -2:0 */
+ vpsrldq $14, RNOT, RNOT; /* ab: -1:0 ; cd: -1:0 */
+ vpaddw RNOT, RNOT, RTMP2; /* ab: -2:0 ; cd: -2:0 */
/* load IV and byteswap */
vmovdqu (%rcx), RTMP4x;
vpshufb RTMP3x, RTMP4x, RTMP4x;
vmovdqa RTMP4x, RTMP0x;
- inc_le128(RTMP4x, RNOTx, RTMP1x);
+ vpsubw RNOTx, RTMP4x, RTMP4x;
vinserti128 $1, RTMP4x, RTMP0, RTMP0;
vpshufb RTMP3, RTMP0, RA0; /* +1 ; +0 */
- /* check need for handling 64-bit overflow and carry */
- cmpq $(0xffffffffffffffff - 16), %rax;
- ja .Lhandle_ctr_carry;
-
/* construct IVs */
- vpsubq RTMP2, RTMP0, RTMP0; /* +3 ; +2 */
+ vpsubw RTMP2, RTMP0, RTMP0; /* +3 ; +2 */
vpshufb RTMP3, RTMP0, RA1;
- vpsubq RTMP2, RTMP0, RTMP0; /* +5 ; +4 */
+ vpsubw RTMP2, RTMP0, RTMP0; /* +5 ; +4 */
vpshufb RTMP3, RTMP0, RA2;
- vpsubq RTMP2, RTMP0, RTMP0; /* +7 ; +6 */
+ vpsubw RTMP2, RTMP0, RTMP0; /* +7 ; +6 */
vpshufb RTMP3, RTMP0, RA3;
- vpsubq RTMP2, RTMP0, RTMP0; /* +9 ; +8 */
+ vpsubw RTMP2, RTMP0, RTMP0; /* +9 ; +8 */
vpshufb RTMP3, RTMP0, RB0;
- vpsubq RTMP2, RTMP0, RTMP0; /* +11 ; +10 */
+ vpsubw RTMP2, RTMP0, RTMP0; /* +11 ; +10 */
vpshufb RTMP3, RTMP0, RB1;
- vpsubq RTMP2, RTMP0, RTMP0; /* +13 ; +12 */
+ vpsubw RTMP2, RTMP0, RTMP0; /* +13 ; +12 */
vpshufb RTMP3, RTMP0, RB2;
- vpsubq RTMP2, RTMP0, RTMP0; /* +15 ; +14 */
+ vpsubw RTMP2, RTMP0, RTMP0; /* +15 ; +14 */
vpshufb RTMP3, RTMP0, RB3;
- vpsubq RTMP2, RTMP0, RTMP0; /* +16 */
- vpshufb RTMP3x, RTMP0x, RTMP0x;
-
- jmp .Lctr_carry_done;
-
-.Lhandle_ctr_carry:
- /* construct IVs */
- inc_le128(RTMP0, RNOT, RTMP1);
- inc_le128(RTMP0, RNOT, RTMP1);
- vpshufb RTMP3, RTMP0, RA1; /* +3 ; +2 */
- inc_le128(RTMP0, RNOT, RTMP1);
- inc_le128(RTMP0, RNOT, RTMP1);
- vpshufb RTMP3, RTMP0, RA2; /* +5 ; +4 */
- inc_le128(RTMP0, RNOT, RTMP1);
- inc_le128(RTMP0, RNOT, RTMP1);
- vpshufb RTMP3, RTMP0, RA3; /* +7 ; +6 */
- inc_le128(RTMP0, RNOT, RTMP1);
- inc_le128(RTMP0, RNOT, RTMP1);
- vpshufb RTMP3, RTMP0, RB0; /* +9 ; +8 */
- inc_le128(RTMP0, RNOT, RTMP1);
- inc_le128(RTMP0, RNOT, RTMP1);
- vpshufb RTMP3, RTMP0, RB1; /* +11 ; +10 */
- inc_le128(RTMP0, RNOT, RTMP1);
- inc_le128(RTMP0, RNOT, RTMP1);
- vpshufb RTMP3, RTMP0, RB2; /* +13 ; +12 */
- inc_le128(RTMP0, RNOT, RTMP1);
- inc_le128(RTMP0, RNOT, RTMP1);
- vpshufb RTMP3, RTMP0, RB3; /* +15 ; +14 */
- inc_le128(RTMP0, RNOT, RTMP1);
- vextracti128 $1, RTMP0, RTMP0x;
- vpshufb RTMP3x, RTMP0x, RTMP0x; /* +16 */
-
-.Lctr_carry_done:
- /* store new IV */
- vmovdqu RTMP0x, (%rcx);
.align 8
.Lload_ctr_done:
+ /* Update counter */
+ addb $16, 15(%rcx);
+ adcb $0, 14(%rcx);
+
call SM4_CRYPT_BLK16;
vpxor (0 * 32)(%rdx), RA0, RA0;
@@ -175,24 +134,8 @@ FUNC_NAME(ctr_enc):
ret_spec_stop;
.align 8
-.Lctr_byteadd_full_ctr_carry:
- movq 8(%rcx), %r11;
- movq (%rcx), %r10;
- bswapq %r11;
- bswapq %r10;
- addq $16, %r11;
- adcq $0, %r10;
- bswapq %r11;
- bswapq %r10;
- movq %r11, 8(%rcx);
- movq %r10, (%rcx);
- jmp .Lctr_byteadd_ymm;
-.align 8
.Lctr_byteadd:
vbroadcasti128 (%rcx), RB3;
- je .Lctr_byteadd_full_ctr_carry;
- addb $16, 15(%rcx);
-.Lctr_byteadd_ymm:
vpaddb .Lbige_addb_0_1 rRIP, RB3, RA0;
vpaddb .Lbige_addb_2_3 rRIP, RB3, RA1;
vpaddb .Lbige_addb_4_5 rRIP, RB3, RA2;
diff --git a/cipher/sm4-avx512-amd64.h b/cipher/sm4-avx512-amd64.h
index 3ee5a4f6..05e032de 100644
--- a/cipher/sm4-avx512-amd64.h
+++ b/cipher/sm4-avx512-amd64.h
@@ -70,8 +70,6 @@ FUNC_NAME(cipher_mode_consts):
.quad 8, 0
.Lcounter16161616_lo:
.quad 16, 0
-.Lcounter1111_hi:
- .quad 0, 1
.text
@@ -138,57 +136,19 @@ FUNC_NAME(ctr_enc_blk32):
vbroadcasti64x2 .Lcounter16161616_lo rRIP, RTMP4z;
/* load IV and byteswap */
- movq 8(%rcx), %r11;
- bswapq %r11;
vbroadcasti64x2 (%rcx), RB3z;
vpshufb RTMP0z, RB3z, RB3z;
- /* check need for handling 64-bit overflow and carry */
- cmpq $(0xffffffffffffffff - 32), %r11;
- ja .Lhandle_ctr_carry_blk32;
-
- /* construct IVs */
- vpaddq RTMP1z, RB3z, RA0z; /* +0:+1:+2:+3 */
- vpaddq RTMP2z, RA0z, RA1z; /* +4:+5:+6:+7 */
- vpaddq RTMP3z, RA0z, RA2z; /* +8:+9:+10:+11 */
- vpaddq RTMP3z, RA1z, RA3z; /* +12:+13:+14:+15 */
- vpaddq RTMP4z, RA0z, RB0z; /* +16... */
- vpaddq RTMP4z, RA1z, RB1z; /* +20... */
- vpaddq RTMP4z, RA2z, RB2z; /* +24... */
- vpaddq RTMP4z, RA3z, RB3z; /* +28... */
-
- /* Update counter */
- leaq 32(%r11), %r11;
- bswapq %r11;
- movq %r11, 8(%rcx);
-
- jmp .Lctr_carry_done_blk32;
-
-.Lhandle_ctr_carry_blk32:
- vbroadcasti64x2 .Lcounter1111_hi rRIP, RNOTz;
-
/* construct IVs */
- add_le128(RA0z, RB3z, RTMP1z, RNOTz); /* +0:+1:+2:+3 */
- add_le128(RA1z, RA0z, RTMP2z, RNOTz); /* +4:+5:+6:+7 */
- add_le128(RA2z, RA0z, RTMP3z, RNOTz); /* +8:+9:+10:+11 */
- add_le128(RA3z, RA1z, RTMP3z, RNOTz); /* +12:+13:+14:+15 */
- add_le128(RB0z, RA0z, RTMP4z, RNOTz); /* +16... */
- add_le128(RB1z, RA1z, RTMP4z, RNOTz); /* +20... */
- add_le128(RB2z, RA2z, RTMP4z, RNOTz); /* +24... */
- add_le128(RB3z, RA3z, RTMP4z, RNOTz); /* +28... */
+ vpaddw RTMP1z, RB3z, RA0z; /* +0:+1:+2:+3 */
+ vpaddw RTMP2z, RA0z, RA1z; /* +4:+5:+6:+7 */
+ vpaddw RTMP3z, RA0z, RA2z; /* +8:+9:+10:+11 */
+ vpaddw RTMP3z, RA1z, RA3z; /* +12:+13:+14:+15 */
+ vpaddw RTMP4z, RA0z, RB0z; /* +16... */
+ vpaddw RTMP4z, RA1z, RB1z; /* +20... */
+ vpaddw RTMP4z, RA2z, RB2z; /* +24... */
+ vpaddw RTMP4z, RA3z, RB3z; /* +28... */
- /* Update counter */
- addq $32, %r11;
- movq (%rcx), %r10;
- bswapq %r10;
- adcq $0, %r10;
- bswapq %r11;
- bswapq %r10;
- movq %r11, 8(%rcx);
- movq %r10, (%rcx);
-
-.align 16
-.Lctr_carry_done_blk32:
/* Byte-swap IVs. */
vpshufb RTMP0z, RA0z, RA0z;
vpshufb RTMP0z, RA1z, RA1z;
@@ -201,6 +161,10 @@ FUNC_NAME(ctr_enc_blk32):
.align 16
.Lload_ctr_done32:
+ /* Update counter */
+ addb $32, 15(%rcx);
+ adcb $0, 14(%rcx);
+
call SM4_CRYPT_BLK32;
vpxord (0 * 64)(%rdx), RA0z, RA0z;
@@ -222,29 +186,12 @@ FUNC_NAME(ctr_enc_blk32):
vmovdqu32 RB3z, (7 * 64)(%rsi);
vzeroall;
- kxorq %k1, %k1, %k1;
ret_spec_stop;
.align 16
-.Lctr_byteadd_full_ctr_carry32:
- movq 8(%rcx), %r11;
- movq (%rcx), %r10;
- bswapq %r11;
- bswapq %r10;
- addq $32, %r11;
- adcq $0, %r10;
- bswapq %r11;
- bswapq %r10;
- movq %r11, 8(%rcx);
- movq %r10, (%rcx);
- jmp .Lctr_byteadd_zmm32;
-.align 16
.Lctr_byteadd32:
vbroadcasti64x2 (%rcx), RA3z;
- je .Lctr_byteadd_full_ctr_carry32;
- addb $32, 15(%rcx);
-.Lctr_byteadd_zmm32:
vbroadcasti64x2 .Lbige_addb_16 rRIP, RB3z;
vpaddb RB3z, RA3z, RB3z;
vpaddb .Lbige_addb_0_1 rRIP, RA3z, RA0z;
diff --git a/cipher/sm4-gfni-avx512-amd64.S b/cipher/sm4-gfni-avx512-amd64.S
index 8bc8bd81..e69b2ab5 100644
--- a/cipher/sm4-gfni-avx512-amd64.S
+++ b/cipher/sm4-gfni-avx512-amd64.S
@@ -590,12 +590,6 @@ _gcry_sm4_gfni_avx512_crypt_blk1_16:
CFI_ENDPROC();
ELF(.size _gcry_sm4_gfni_avx512_crypt_blk1_16,.-_gcry_sm4_gfni_avx512_crypt_blk1_16;)
-#define add_le128(out, in, lo_counter, hi_counter1) \
- vpaddq lo_counter, in, out; \
- vpcmpuq $1, lo_counter, out, %k1; \
- kaddb %k1, %k1, %k1; \
- vpaddq hi_counter1, out, out{%k1};
-
.align 16
.globl _gcry_sm4_gfni_avx512_ctr_enc
ELF(.type _gcry_sm4_gfni_avx512_ctr_enc, at function;)
@@ -619,57 +613,19 @@ _gcry_sm4_gfni_avx512_ctr_enc:
vbroadcasti128 .Lcounter8888_lo rRIP, RTMP4;
/* load IV and byteswap */
- movq 8(%rcx), %r11;
- bswapq %r11;
vbroadcasti128 (%rcx), RB3;
vpshufb RTMP0, RB3, RB3;
- /* check need for handling 64-bit overflow and carry */
- cmpq $(0xffffffffffffffff - 16), %r11;
- ja .Lhandle_ctr_carry_blk16;
-
- /* construct IVs */
- vpaddq RTMP1, RB3, RA0; /* +0:+1 */
- vpaddq RTMP2, RA0, RA1; /* +2:+3 */
- vpaddq RTMP3, RA0, RA2; /* +4:+5 */
- vpaddq RTMP3, RA1, RA3; /* +6:+7 */
- vpaddq RTMP4, RA0, RB0; /* +8... */
- vpaddq RTMP4, RA1, RB1; /* +10... */
- vpaddq RTMP4, RA2, RB2; /* +12... */
- vpaddq RTMP4, RA3, RB3; /* +14... */
-
- /* Update counter */
- leaq 16(%r11), %r11;
- bswapq %r11;
- movq %r11, 8(%rcx);
-
- jmp .Lctr_carry_done_blk16;
-
-.Lhandle_ctr_carry_blk16:
- vbroadcasti128 .Lcounter1111_hi rRIP, RNOT;
-
/* construct IVs */
- add_le128(RA0, RB3, RTMP1, RNOT); /* +0:+1 */
- add_le128(RA1, RA0, RTMP2, RNOT); /* +2:+3 */
- add_le128(RA2, RA0, RTMP3, RNOT); /* +4:+5 */
- add_le128(RA3, RA1, RTMP3, RNOT); /* +6:+7 */
- add_le128(RB0, RA0, RTMP4, RNOT); /* +8... */
- add_le128(RB1, RA1, RTMP4, RNOT); /* +10... */
- add_le128(RB2, RA2, RTMP4, RNOT); /* +12... */
- add_le128(RB3, RA3, RTMP4, RNOT); /* +14... */
+ vpaddw RTMP1, RB3, RA0; /* +0:+1 */
+ vpaddw RTMP2, RA0, RA1; /* +2:+3 */
+ vpaddw RTMP3, RA0, RA2; /* +4:+5 */
+ vpaddw RTMP3, RA1, RA3; /* +6:+7 */
+ vpaddw RTMP4, RA0, RB0; /* +8... */
+ vpaddw RTMP4, RA1, RB1; /* +10... */
+ vpaddw RTMP4, RA2, RB2; /* +12... */
+ vpaddw RTMP4, RA3, RB3; /* +14... */
- /* Update counter */
- addq $16, %r11;
- movq (%rcx), %r10;
- bswapq %r10;
- adcq $0, %r10;
- bswapq %r11;
- bswapq %r10;
- movq %r11, 8(%rcx);
- movq %r10, (%rcx);
-
-.align 16
-.Lctr_carry_done_blk16:
/* Byte-swap IVs. */
vpshufb RTMP0, RA0, RA0;
vpshufb RTMP0, RA1, RA1;
@@ -682,6 +638,10 @@ _gcry_sm4_gfni_avx512_ctr_enc:
.align 16
.Lload_ctr_done16:
+ /* Update counter */
+ addb $16, 15(%rcx);
+ adcb $0, 14(%rcx);
+
call __sm4_gfni_crypt_blk16;
vpxor (0 * 32)(%rdx), RA0, RA0;
@@ -703,29 +663,12 @@ _gcry_sm4_gfni_avx512_ctr_enc:
vmovdqu RB3, (7 * 32)(%rsi);
vzeroall;
- kxorq %k1, %k1, %k1;
ret_spec_stop;
.align 16
-.Lctr_byteadd_full_ctr_carry16:
- movq 8(%rcx), %r11;
- movq (%rcx), %r10;
- bswapq %r11;
- bswapq %r10;
- addq $16, %r11;
- adcq $0, %r10;
- bswapq %r11;
- bswapq %r10;
- movq %r11, 8(%rcx);
- movq %r10, (%rcx);
- jmp .Lctr_byteadd_ymm16;
-.align 16
.Lctr_byteadd16:
vbroadcasti128 (%rcx), RB3;
- je .Lctr_byteadd_full_ctr_carry16;
- addb $16, 15(%rcx);
-.Lctr_byteadd_ymm16:
vpaddb .Lbige_addb_0_1 rRIP, RB3, RA0;
vpaddb .Lbige_addb_2_3 rRIP, RB3, RA1;
vpaddb .Lbige_addb_4_5 rRIP, RB3, RA2;
diff --git a/cipher/sm4-intel-avx512-amd64.S b/cipher/sm4-intel-avx512-amd64.S
index a4963db0..c5eaa723 100644
--- a/cipher/sm4-intel-avx512-amd64.S
+++ b/cipher/sm4-intel-avx512-amd64.S
@@ -160,12 +160,6 @@ __sm4_intel_crypt_blk32:
CFI_ENDPROC();
ELF(.size __sm4_intel_crypt_blk32,.-__sm4_intel_crypt_blk32;)
-#define add_le128(out, in, lo_counter, hi_counter1) \
- vpaddq lo_counter, in, out; \
- vpcmpuq $1, lo_counter, out, %k1; \
- kaddb %k1, %k1, %k1; \
- vpaddq hi_counter1, out, out{%k1};
-
#define FUNC_NAME(func) _gcry_sm4_intel_avx512_ ## func
#define SM4_CRYPT_BLK32 __sm4_intel_crypt_blk32
#include "sm4-avx512-amd64.h"
diff --git a/cipher/sm4.c b/cipher/sm4.c
index 70b4901c..482a7983 100644
--- a/cipher/sm4.c
+++ b/cipher/sm4.c
@@ -995,7 +995,7 @@ sm4_setkey (void *context, const byte *key, const unsigned keylen,
memset (bulk_ops, 0, sizeof(*bulk_ops));
bulk_ops->cbc_dec = _gcry_sm4_cbc_dec;
bulk_ops->cfb_dec = _gcry_sm4_cfb_dec;
- bulk_ops->ctr_enc = _gcry_sm4_ctr_enc;
+ bulk_ops->ctr16be_enc = _gcry_sm4_ctr_enc;
bulk_ops->xts_crypt = _gcry_sm4_xts_crypt;
bulk_ops->ecb_crypt = _gcry_sm4_ecb_crypt;
bulk_ops->ctr32le_enc = _gcry_sm4_ctr32le_enc;
--
2.53.0
More information about the Gcrypt-devel
mailing list