[PATCH 02/10] aria: reduce CTR bulk counter carry handling to 16 bits
Jussi Kivilinna
jussi.kivilinna at iki.fi
Fri Jul 24 20:50:09 CEST 2026
* cipher/aria.c (aria_setkey): Assign ctr16be_enc bulk op.
* cipher/aria-aesni-avx-amd64.S (inc_le128): Remove.
(__aria_aesni_avx_ctr_gen_keystream_16way): Add to low 16 counter bits
only, drop full-width overflow path.
* cipher/aria-aesni-avx2-amd64.S: Likewise.
* cipher/aria-gfni-avx512-amd64.S: Likewise.
--
ARIA CTR keystream generators did full 128-bit little-endian counter
increment with separate 64-bit overflow path. Under ctr16be_enc
contract generic ctr code splits work at low 16-bit overflow, so bulk
function adds only to low 16 counter bits. Drop inc_le128 and overflow
path, use 16-bit stepping.
Moves ARIA off transitional ctr_enc alias.
Signed-off-by: Jussi Kivilinna <jussi.kivilinna at iki.fi>
---
cipher/aria-aesni-avx-amd64.S | 86 ++++++----------
cipher/aria-aesni-avx2-amd64.S | 170 ++++++--------------------------
cipher/aria-gfni-avx512-amd64.S | 107 +++++---------------
cipher/aria.c | 2 +-
4 files changed, 86 insertions(+), 279 deletions(-)
diff --git a/cipher/aria-aesni-avx-amd64.S b/cipher/aria-aesni-avx-amd64.S
index 29597488..b177ca5e 100644
--- a/cipher/aria-aesni-avx-amd64.S
+++ b/cipher/aria-aesni-avx-amd64.S
@@ -66,12 +66,6 @@
((l0) << (7 * 8)) )
/* asm macros */
-#define inc_le128(x, minus_one, tmp) \
- vpcmpeqq minus_one, x, tmp; \
- vpsubq minus_one, x, x; \
- vpslldq $8, tmp, tmp; \
- vpsubq tmp, x, x;
-
#define filter_8bit(x, lo_t, hi_t, mask4bit, tmp0) \
vpand x, mask4bit, tmp0; \
vpandn x, mask4bit, x; \
@@ -1097,82 +1091,52 @@ __aria_aesni_avx_ctr_gen_keystream_16way:
vpshufb %xmm1, %xmm8, %xmm3; /* be => le */
vpcmpeqd %xmm0, %xmm0, %xmm0;
- vpsrldq $8, %xmm0, %xmm0; /* low: -1, high: 0 */
+ vpsrldq $14, %xmm0, %xmm0; /* low: -1, high: 0 */
/* construct IVs */
- inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
+ vpsubw %xmm0, %xmm3, %xmm3; /* +1 */
vpshufb %xmm1, %xmm3, %xmm9;
- inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
- vpshufb %xmm1, %xmm3, %xmm10;
- inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
+ vpsubw %xmm0, %xmm3, %xmm3; /* +1 */
+ vpshufb %xmm1, %xmm3, %xmm2;
+ vpsubw %xmm0, %xmm3, %xmm3; /* +1 */
vpshufb %xmm1, %xmm3, %xmm11;
- inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
- vpshufb %xmm1, %xmm3, %xmm12;
- inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
- vpshufb %xmm1, %xmm3, %xmm13;
- inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
- vpshufb %xmm1, %xmm3, %xmm14;
- inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
- vpshufb %xmm1, %xmm3, %xmm15;
+ vpsubw %xmm0, %xmm3, %xmm3; /* +1 */
+ vpshufb %xmm1, %xmm3, %xmm4;
+ vpsubw %xmm0, %xmm3, %xmm3; /* +1 */
+ vpshufb %xmm1, %xmm3, %xmm5;
+ vpsubw %xmm0, %xmm3, %xmm3; /* +1 */
+ vpshufb %xmm1, %xmm3, %xmm6;
+ vpsubw %xmm0, %xmm3, %xmm3; /* +1 */
+ vpshufb %xmm1, %xmm3, %xmm7;
vmovdqu %xmm8, (0 * 16)(%rcx);
vmovdqu %xmm9, (1 * 16)(%rcx);
- vmovdqu %xmm10, (2 * 16)(%rcx);
vmovdqu %xmm11, (3 * 16)(%rcx);
- vmovdqu %xmm12, (4 * 16)(%rcx);
- vmovdqu %xmm13, (5 * 16)(%rcx);
- vmovdqu %xmm14, (6 * 16)(%rcx);
- vmovdqu %xmm15, (7 * 16)(%rcx);
- inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
+ vpsubw %xmm0, %xmm3, %xmm3; /* +1 */
vpshufb %xmm1, %xmm3, %xmm8;
- inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
+ vpsubw %xmm0, %xmm3, %xmm3; /* +1 */
vpshufb %xmm1, %xmm3, %xmm9;
- inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
+ vpsubw %xmm0, %xmm3, %xmm3; /* +1 */
vpshufb %xmm1, %xmm3, %xmm10;
- inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
+ vpsubw %xmm0, %xmm3, %xmm3; /* +1 */
vpshufb %xmm1, %xmm3, %xmm11;
- inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
+ vpsubw %xmm0, %xmm3, %xmm3; /* +1 */
vpshufb %xmm1, %xmm3, %xmm12;
- inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
+ vpsubw %xmm0, %xmm3, %xmm3; /* +1 */
vpshufb %xmm1, %xmm3, %xmm13;
- inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
+ vpsubw %xmm0, %xmm3, %xmm3; /* +1 */
vpshufb %xmm1, %xmm3, %xmm14;
- inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
+ vpsubw %xmm0, %xmm3, %xmm3; /* +1 */
vpshufb %xmm1, %xmm3, %xmm15;
- inc_le128(%xmm3, %xmm0, %xmm5); /* +1 */
- vpshufb %xmm1, %xmm3, %xmm4;
- vmovdqu %xmm4, (%r8);
vmovdqu (0 * 16)(%rcx), %xmm0;
vmovdqu (1 * 16)(%rcx), %xmm1;
- vmovdqu (2 * 16)(%rcx), %xmm2;
vmovdqu (3 * 16)(%rcx), %xmm3;
- vmovdqu (4 * 16)(%rcx), %xmm4;
- vmovdqu (5 * 16)(%rcx), %xmm5;
- vmovdqu (6 * 16)(%rcx), %xmm6;
- vmovdqu (7 * 16)(%rcx), %xmm7;
- ret_spec_stop;
+ jmp .Lctr_gen_keystream_done;
.align 8
-.Lctr_byteadd_full_ctr_carry:
- addb $16, 15(%r8);
- pushq %rcx;
- CFI_ADJUST_CFA_OFFSET(8);
- movl $14, %ecx;
- 1:
- adcb $0, (%r8, %rcx);
- jnc 2f;
- loop 1b;
- 2:
- popq %rcx;
- CFI_ADJUST_CFA_OFFSET(-8);
- jmp .Lctr_byteadd_xmm;
-.align 8
.Lctr_byteadd:
- je .Lctr_byteadd_full_ctr_carry;
- addb $16, 15(%r8);
-.Lctr_byteadd_xmm:
vmovdqa %xmm8, %xmm0;
vpaddb .Lbige_addb_1 rRIP, %xmm8, %xmm1;
vpaddb .Lbige_addb_2 rRIP, %xmm8, %xmm2;
@@ -1190,6 +1154,12 @@ __aria_aesni_avx_ctr_gen_keystream_16way:
vpaddb .Lbige_addb_14 rRIP, %xmm0, %xmm14;
vpaddb .Lbige_addb_15 rRIP, %xmm0, %xmm15;
+.align 16
+.Lctr_gen_keystream_done:
+ /* Update IV */
+ addb $16, 15(%r8);
+ adcb $0, 14(%r8);
+
ret_spec_stop;
CFI_ENDPROC();
ELF(.size __aria_aesni_avx_ctr_gen_keystream_16way,.-__aria_aesni_avx_ctr_gen_keystream_16way;)
diff --git a/cipher/aria-aesni-avx2-amd64.S b/cipher/aria-aesni-avx2-amd64.S
index 08bf5a0e..99eca542 100644
--- a/cipher/aria-aesni-avx2-amd64.S
+++ b/cipher/aria-aesni-avx2-amd64.S
@@ -86,12 +86,6 @@
((l0) << (7 * 8)) )
/* asm macros */
-#define inc_le128(x, minus_one, tmp) \
- vpcmpeqq minus_one, x, tmp; \
- vpsubq minus_one, x, x; \
- vpslldq $8, tmp, tmp; \
- vpsubq tmp, x, x;
-
#define filter_8bit(x, lo_t, hi_t, mask4bit, tmp0) \
vpand x, mask4bit, tmp0; \
vpandn x, mask4bit, x; \
@@ -1246,171 +1240,63 @@ __aria_aesni_avx2_ctr_gen_keystream_32way:
cmpb $(0x100 - 32), 15(%r8);
jbe .Lctr_byteadd;
- movq 8(%r8), %r11;
- bswapq %r11;
-
vbroadcasti128 .Lbswap128_mask rRIP, %ymm6;
vpcmpeqd %ymm0, %ymm0, %ymm0;
- vpsrldq $8, %ymm0, %ymm0; /* ab: -1:0 ; cd: -1:0 */
- vpaddq %ymm0, %ymm0, %ymm5; /* ab: -2:0 ; cd: -2:0 */
+ vpsrldq $14, %ymm0, %ymm0; /* ab: -1:0 ; cd: -1:0 */
+ vpaddw %ymm0, %ymm0, %ymm5; /* ab: -2:0 ; cd: -2:0 */
- /* load IV and byteswap */
+ /* load IV and byteswap. */
vmovdqu (%r8), %xmm7;
vpshufb %xmm6, %xmm7, %xmm7;
vmovdqa %xmm7, %xmm3;
- inc_le128(%xmm7, %xmm0, %xmm4);
+ vpsubw %xmm0, %xmm7, %xmm7;
vinserti128 $1, %xmm7, %ymm3, %ymm3;
- vpshufb %ymm6, %ymm3, %ymm8; /* +1 ; +0 */
-
- /* check need for handling 64-bit overflow and carry */
- cmpq $(0xffffffffffffffff - 32), %r11;
- ja .Lhandle_ctr_carry;
+ vpshufb %ymm6, %ymm3, %ymm0; /* +1 ; +0 */
/* construct IVs */
- vpsubq %ymm5, %ymm3, %ymm3; /* +3 ; +2 */
- vpshufb %ymm6, %ymm3, %ymm9;
- vpsubq %ymm5, %ymm3, %ymm3; /* +5 ; +4 */
- vpshufb %ymm6, %ymm3, %ymm10;
- vpsubq %ymm5, %ymm3, %ymm3; /* +7 ; +6 */
+ vpsubw %ymm5, %ymm3, %ymm3; /* +3 ; +2 */
+ vpshufb %ymm6, %ymm3, %ymm1;
+ vpsubw %ymm5, %ymm3, %ymm3; /* +5 ; +4 */
+ vpshufb %ymm6, %ymm3, %ymm2;
+ vpsubw %ymm5, %ymm3, %ymm3; /* +7 ; +6 */
vpshufb %ymm6, %ymm3, %ymm11;
- vpsubq %ymm5, %ymm3, %ymm3; /* +9 ; +8 */
- vpshufb %ymm6, %ymm3, %ymm12;
- vpsubq %ymm5, %ymm3, %ymm3; /* +11 ; +10 */
+ vpsubw %ymm5, %ymm3, %ymm3; /* +9 ; +8 */
+ vpshufb %ymm6, %ymm3, %ymm4;
+ vpsubw %ymm5, %ymm3, %ymm3; /* +11 ; +10 */
vpshufb %ymm6, %ymm3, %ymm13;
- vpsubq %ymm5, %ymm3, %ymm3; /* +13 ; +12 */
+ vpsubw %ymm5, %ymm3, %ymm3; /* +13 ; +12 */
vpshufb %ymm6, %ymm3, %ymm14;
- vpsubq %ymm5, %ymm3, %ymm3; /* +15 ; +14 */
- vpshufb %ymm6, %ymm3, %ymm15;
- vmovdqu %ymm8, (0 * 32)(%rcx);
- vmovdqu %ymm9, (1 * 32)(%rcx);
- vmovdqu %ymm10, (2 * 32)(%rcx);
+ vpsubw %ymm5, %ymm3, %ymm3; /* +15 ; +14 */
+ vpshufb %ymm6, %ymm3, %ymm7;
vmovdqu %ymm11, (3 * 32)(%rcx);
- vmovdqu %ymm12, (4 * 32)(%rcx);
vmovdqu %ymm13, (5 * 32)(%rcx);
vmovdqu %ymm14, (6 * 32)(%rcx);
- vmovdqu %ymm15, (7 * 32)(%rcx);
- vpsubq %ymm5, %ymm3, %ymm3; /* +17 ; +16 */
+ vpsubw %ymm5, %ymm3, %ymm3; /* +17 ; +16 */
vpshufb %ymm6, %ymm3, %ymm8;
- vpsubq %ymm5, %ymm3, %ymm3; /* +19 ; +18 */
+ vpsubw %ymm5, %ymm3, %ymm3; /* +19 ; +18 */
vpshufb %ymm6, %ymm3, %ymm9;
- vpsubq %ymm5, %ymm3, %ymm3; /* +21 ; +20 */
+ vpsubw %ymm5, %ymm3, %ymm3; /* +21 ; +20 */
vpshufb %ymm6, %ymm3, %ymm10;
- vpsubq %ymm5, %ymm3, %ymm3; /* +23 ; +22 */
+ vpsubw %ymm5, %ymm3, %ymm3; /* +23 ; +22 */
vpshufb %ymm6, %ymm3, %ymm11;
- vpsubq %ymm5, %ymm3, %ymm3; /* +25 ; +24 */
+ vpsubw %ymm5, %ymm3, %ymm3; /* +25 ; +24 */
vpshufb %ymm6, %ymm3, %ymm12;
- vpsubq %ymm5, %ymm3, %ymm3; /* +27 ; +26 */
+ vpsubw %ymm5, %ymm3, %ymm3; /* +27 ; +26 */
vpshufb %ymm6, %ymm3, %ymm13;
- vpsubq %ymm5, %ymm3, %ymm3; /* +29 ; +28 */
+ vpsubw %ymm5, %ymm3, %ymm3; /* +29 ; +28 */
vpshufb %ymm6, %ymm3, %ymm14;
- vpsubq %ymm5, %ymm3, %ymm3; /* +31 ; +30 */
+ vpsubw %ymm5, %ymm3, %ymm3; /* +31 ; +30 */
vpshufb %ymm6, %ymm3, %ymm15;
- vpsubq %ymm5, %ymm3, %ymm3; /* +32 */
- vpshufb %xmm6, %xmm3, %xmm3;
- vmovdqu %xmm3, (%r8);
- vmovdqu (0 * 32)(%rcx), %ymm0;
- vmovdqu (1 * 32)(%rcx), %ymm1;
- vmovdqu (2 * 32)(%rcx), %ymm2;
- vmovdqu (3 * 32)(%rcx), %ymm3;
- vmovdqu (4 * 32)(%rcx), %ymm4;
- vmovdqu (5 * 32)(%rcx), %ymm5;
- vmovdqu (6 * 32)(%rcx), %ymm6;
- vmovdqu (7 * 32)(%rcx), %ymm7;
- jmp .Lctr_carry_done;
-
- .Lhandle_ctr_carry:
- /* construct IVs */
- inc_le128(%ymm3, %ymm0, %ymm4);
- inc_le128(%ymm3, %ymm0, %ymm4);
- vpshufb %ymm6, %ymm3, %ymm9; /* +3 ; +2 */
- inc_le128(%ymm3, %ymm0, %ymm4);
- inc_le128(%ymm3, %ymm0, %ymm4);
- vpshufb %ymm6, %ymm3, %ymm10; /* +5 ; +4 */
- inc_le128(%ymm3, %ymm0, %ymm4);
- inc_le128(%ymm3, %ymm0, %ymm4);
- vpshufb %ymm6, %ymm3, %ymm11; /* +7 ; +6 */
- inc_le128(%ymm3, %ymm0, %ymm4);
- inc_le128(%ymm3, %ymm0, %ymm4);
- vpshufb %ymm6, %ymm3, %ymm12; /* +9 ; +8 */
- inc_le128(%ymm3, %ymm0, %ymm4);
- inc_le128(%ymm3, %ymm0, %ymm4);
- vpshufb %ymm6, %ymm3, %ymm13; /* +11 ; +10 */
- inc_le128(%ymm3, %ymm0, %ymm4);
- inc_le128(%ymm3, %ymm0, %ymm4);
- vpshufb %ymm6, %ymm3, %ymm14; /* +13 ; +12 */
- inc_le128(%ymm3, %ymm0, %ymm4);
- inc_le128(%ymm3, %ymm0, %ymm4);
- vpshufb %ymm6, %ymm3, %ymm15; /* +15 ; +14 */
- vmovdqu %ymm8, (0 * 32)(%rcx);
- vmovdqu %ymm9, (1 * 32)(%rcx);
- vmovdqu %ymm10, (2 * 32)(%rcx);
- vmovdqu %ymm11, (3 * 32)(%rcx);
- vmovdqu %ymm12, (4 * 32)(%rcx);
- vmovdqu %ymm13, (5 * 32)(%rcx);
- vmovdqu %ymm14, (6 * 32)(%rcx);
- vmovdqu %ymm15, (7 * 32)(%rcx);
-
- inc_le128(%ymm3, %ymm0, %ymm4);
- inc_le128(%ymm3, %ymm0, %ymm4);
- vpshufb %ymm6, %ymm3, %ymm8; /* +17 ; +16 */
- inc_le128(%ymm3, %ymm0, %ymm4);
- inc_le128(%ymm3, %ymm0, %ymm4);
- vpshufb %ymm6, %ymm3, %ymm9; /* +19 ; +18 */
- inc_le128(%ymm3, %ymm0, %ymm4);
- inc_le128(%ymm3, %ymm0, %ymm4);
- vpshufb %ymm6, %ymm3, %ymm10; /* +21 ; +20 */
- inc_le128(%ymm3, %ymm0, %ymm4);
- inc_le128(%ymm3, %ymm0, %ymm4);
- vpshufb %ymm6, %ymm3, %ymm11; /* +23 ; +22 */
- inc_le128(%ymm3, %ymm0, %ymm4);
- inc_le128(%ymm3, %ymm0, %ymm4);
- vpshufb %ymm6, %ymm3, %ymm12; /* +25 ; +24 */
- inc_le128(%ymm3, %ymm0, %ymm4);
- inc_le128(%ymm3, %ymm0, %ymm4);
- vpshufb %ymm6, %ymm3, %ymm13; /* +27 ; +26 */
- inc_le128(%ymm3, %ymm0, %ymm4);
- inc_le128(%ymm3, %ymm0, %ymm4);
- vpshufb %ymm6, %ymm3, %ymm14; /* +29 ; +28 */
- inc_le128(%ymm3, %ymm0, %ymm4);
- inc_le128(%ymm3, %ymm0, %ymm4);
- vpshufb %ymm6, %ymm3, %ymm15; /* +31 ; +30 */
- inc_le128(%ymm3, %ymm0, %ymm4);
- vextracti128 $1, %ymm3, %xmm3;
- vpshufb %xmm6, %xmm3, %xmm3; /* +32 */
- vmovdqu %xmm3, (%r8);
- vmovdqu (0 * 32)(%rcx), %ymm0;
- vmovdqu (1 * 32)(%rcx), %ymm1;
- vmovdqu (2 * 32)(%rcx), %ymm2;
vmovdqu (3 * 32)(%rcx), %ymm3;
- vmovdqu (4 * 32)(%rcx), %ymm4;
vmovdqu (5 * 32)(%rcx), %ymm5;
vmovdqu (6 * 32)(%rcx), %ymm6;
- vmovdqu (7 * 32)(%rcx), %ymm7;
-.Lctr_carry_done:
- ret_spec_stop;
+ jmp .Lctr_gen_keystream_done;
.align 8
-.Lctr_byteadd_full_ctr_carry:
- addb $32, 15(%r8);
- pushq %rcx;
- CFI_ADJUST_CFA_OFFSET(8);
- movl $14, %ecx;
- 1:
- adcb $0, (%r8, %rcx);
- jnc 2f;
- loop 1b;
- 2:
- popq %rcx;
- CFI_ADJUST_CFA_OFFSET(-8);
- jmp .Lctr_byteadd_ymm;
-.align 8
.Lctr_byteadd:
vbroadcasti128 (%r8), %ymm8;
- je .Lctr_byteadd_full_ctr_carry;
- addb $32, 15(%r8);
-.Lctr_byteadd_ymm:
vpaddb .Lbige_addb_16_16 rRIP, %ymm8, %ymm15;
vpaddb .Lbige_addb_0_1 rRIP, %ymm8, %ymm0;
vpaddb .Lbige_addb_2_3 rRIP, %ymm8, %ymm1;
@@ -1429,6 +1315,12 @@ __aria_aesni_avx2_ctr_gen_keystream_32way:
vpaddb .Lbige_addb_12_13 rRIP, %ymm15, %ymm14;
vpaddb .Lbige_addb_14_15 rRIP, %ymm15, %ymm15;
+.align 16
+.Lctr_gen_keystream_done:
+ /* Update IV */
+ addb $32, 15(%r8);
+ adcb $0, 14(%r8);
+
ret_spec_stop;
CFI_ENDPROC();
ELF(.size __aria_aesni_avx2_ctr_gen_keystream_32way,
diff --git a/cipher/aria-gfni-avx512-amd64.S b/cipher/aria-gfni-avx512-amd64.S
index a497b395..c25caf86 100644
--- a/cipher/aria-gfni-avx512-amd64.S
+++ b/cipher/aria-gfni-avx512-amd64.S
@@ -75,16 +75,9 @@
clear_vec4(%ymm19, %ymm23, %ymm27, %ymm31)
#define clear_regs() \
- kxorq %k1, %k1, %k1; \
vzeroall; \
clear_zmm16_zmm31()
-#define add_le128(out, in, lo_counter, hi_counter1) \
- vpaddq lo_counter, in, out; \
- vpcmpuq $1, lo_counter, out, %k1; \
- kaddb %k1, %k1, %k1; \
- vpaddq hi_counter1, out, out{%k1};
-
#define filter_8bit(x, lo_t, hi_t, mask4bit, tmp0) \
vpandq x, mask4bit, tmp0; \
vpandqn x, mask4bit, x; \
@@ -583,8 +576,6 @@ SECTION_RODATA
.quad 8, 0
.Lcounter16161616_lo:
.quad 16, 0
-.Lcounter1111_hi:
- .quad 0, 1
/* For CTR-mode IV byteswap */
.Lbswap128_mask:
@@ -815,62 +806,30 @@ __aria_gfni_avx512_ctr_gen_keystream_64way:
vbroadcasti64x2 .Lcounter4444_lo rRIP, %zmm22;
vbroadcasti64x2 .Lcounter8888_lo rRIP, %zmm23;
vbroadcasti64x2 .Lcounter16161616_lo rRIP, %zmm24;
- vbroadcasti64x2 .Lcounter1111_hi rRIP, %zmm25;
- /* load IV and byteswap */
- movq 8(%r8), %r11;
- movq (%r8), %r10;
- bswapq %r11;
- bswapq %r10;
+ /* load IV and byteswap. */
vbroadcasti64x2 (%r8), %zmm20;
vpshufb %zmm19, %zmm20, %zmm20;
- /* check need for handling 64-bit overflow and carry */
- cmpq $(0xffffffffffffffff - 64), %r11;
- ja .Lload_ctr_carry;
-
- /* construct IVs */
- vpaddq %zmm21, %zmm20, %zmm0; /* +0:+1:+2:+3 */
- vpaddq %zmm22, %zmm0, %zmm1; /* +4:+5:+6:+7 */
- vpaddq %zmm23, %zmm0, %zmm2; /* +8:+9:+10:+11 */
- vpaddq %zmm23, %zmm1, %zmm3; /* +12:+13:+14:+15 */
- vpaddq %zmm24, %zmm0, %zmm4; /* +16... */
- vpaddq %zmm24, %zmm1, %zmm5; /* +20... */
- vpaddq %zmm24, %zmm2, %zmm6; /* +24... */
- vpaddq %zmm24, %zmm3, %zmm7; /* +28... */
- vpaddq %zmm24, %zmm4, %zmm8; /* +32... */
- vpaddq %zmm24, %zmm5, %zmm9; /* +36... */
- vpaddq %zmm24, %zmm6, %zmm10; /* +40... */
- vpaddq %zmm24, %zmm7, %zmm11; /* +44... */
- vpaddq %zmm24, %zmm8, %zmm12; /* +48... */
- vpaddq %zmm24, %zmm9, %zmm13; /* +52... */
- vpaddq %zmm24, %zmm10, %zmm14; /* +56... */
- vpaddq %zmm24, %zmm11, %zmm15; /* +60... */
- jmp .Lload_ctr_done;
-
-.Lload_ctr_carry:
/* construct IVs */
- add_le128(%zmm0, %zmm20, %zmm21, %zmm25); /* +0:+1:+2:+3 */
- add_le128(%zmm1, %zmm0, %zmm22, %zmm25); /* +4:+5:+6:+7 */
- add_le128(%zmm2, %zmm0, %zmm23, %zmm25); /* +8:+9:+10:+11 */
- add_le128(%zmm3, %zmm1, %zmm23, %zmm25); /* +12:+13:+14:+15 */
- add_le128(%zmm4, %zmm0, %zmm24, %zmm25); /* +16... */
- add_le128(%zmm5, %zmm1, %zmm24, %zmm25); /* +20... */
- add_le128(%zmm6, %zmm2, %zmm24, %zmm25); /* +24... */
- add_le128(%zmm7, %zmm3, %zmm24, %zmm25); /* +28... */
- add_le128(%zmm8, %zmm4, %zmm24, %zmm25); /* +32... */
- add_le128(%zmm9, %zmm5, %zmm24, %zmm25); /* +36... */
- add_le128(%zmm10, %zmm6, %zmm24, %zmm25); /* +40... */
- add_le128(%zmm11, %zmm7, %zmm24, %zmm25); /* +44... */
- add_le128(%zmm12, %zmm8, %zmm24, %zmm25); /* +48... */
- add_le128(%zmm13, %zmm9, %zmm24, %zmm25); /* +52... */
- add_le128(%zmm14, %zmm10, %zmm24, %zmm25); /* +56... */
- add_le128(%zmm15, %zmm11, %zmm24, %zmm25); /* +60... */
-
-.Lload_ctr_done:
- /* Byte-swap IVs and update counter. */
- addq $64, %r11;
- adcq $0, %r10;
+ vpaddw %zmm21, %zmm20, %zmm0; /* +0:+1:+2:+3 */
+ vpaddw %zmm22, %zmm0, %zmm1; /* +4:+5:+6:+7 */
+ vpaddw %zmm23, %zmm0, %zmm2; /* +8:+9:+10:+11 */
+ vpaddw %zmm23, %zmm1, %zmm3; /* +12:+13:+14:+15 */
+ vpaddw %zmm24, %zmm0, %zmm4; /* +16... */
+ vpaddw %zmm24, %zmm1, %zmm5; /* +20... */
+ vpaddw %zmm24, %zmm2, %zmm6; /* +24... */
+ vpaddw %zmm24, %zmm3, %zmm7; /* +28... */
+ vpaddw %zmm24, %zmm4, %zmm8; /* +32... */
+ vpaddw %zmm24, %zmm5, %zmm9; /* +36... */
+ vpaddw %zmm24, %zmm6, %zmm10; /* +40... */
+ vpaddw %zmm24, %zmm7, %zmm11; /* +44... */
+ vpaddw %zmm24, %zmm8, %zmm12; /* +48... */
+ vpaddw %zmm24, %zmm9, %zmm13; /* +52... */
+ vpaddw %zmm24, %zmm10, %zmm14; /* +56... */
+ vpaddw %zmm24, %zmm11, %zmm15; /* +60... */
+
+ /* Byte-swap IVs. */
vpshufb %zmm19, %zmm15, %zmm15;
vpshufb %zmm19, %zmm14, %zmm14;
vpshufb %zmm19, %zmm13, %zmm13;
@@ -879,8 +838,6 @@ __aria_gfni_avx512_ctr_gen_keystream_64way:
vpshufb %zmm19, %zmm10, %zmm10;
vpshufb %zmm19, %zmm9, %zmm9;
vpshufb %zmm19, %zmm8, %zmm8;
- bswapq %r11;
- bswapq %r10;
vpshufb %zmm19, %zmm7, %zmm7;
vpshufb %zmm19, %zmm6, %zmm6;
vpshufb %zmm19, %zmm5, %zmm5;
@@ -889,30 +846,12 @@ __aria_gfni_avx512_ctr_gen_keystream_64way:
vpshufb %zmm19, %zmm2, %zmm2;
vpshufb %zmm19, %zmm1, %zmm1;
vpshufb %zmm19, %zmm0, %zmm0;
- movq %r11, 8(%r8);
- movq %r10, (%r8);
- ret_spec_stop;
+ jmp .Lctr_gen_keystream_done;
.align 16
-.Lctr_byteadd_full_ctr_carry:
- movq 8(%r8), %r11;
- movq (%r8), %r10;
- bswapq %r11;
- bswapq %r10;
- addq $64, %r11;
- adcq $0, %r10;
- bswapq %r11;
- bswapq %r10;
- movq %r11, 8(%r8);
- movq %r10, (%r8);
- jmp .Lctr_byteadd_zmm;
-.align 16
.Lctr_byteadd:
vbroadcasti64x2 (%r8), %zmm3;
- je .Lctr_byteadd_full_ctr_carry;
- addb $64, 15(%r8);
-.Lctr_byteadd_zmm:
vbroadcasti64x2 .Lbige_addb_16 rRIP, %zmm16;
vmovdqa64 .Lbige_addb_0_1 rRIP, %zmm17;
vmovdqa64 .Lbige_addb_4_5 rRIP, %zmm18;
@@ -938,6 +877,12 @@ __aria_gfni_avx512_ctr_gen_keystream_64way:
vpaddb %zmm19, %zmm15, %zmm14;
vpaddb %zmm20, %zmm15, %zmm15;
+.align 16
+.Lctr_gen_keystream_done:
+ /* Update IV */
+ addb $64, 15(%r8);
+ adcb $0, 14(%r8);
+
ret_spec_stop;
CFI_ENDPROC();
ELF(.size __aria_gfni_avx512_ctr_gen_keystream_64way,
diff --git a/cipher/aria.c b/cipher/aria.c
index bb67ed03..227affb4 100644
--- a/cipher/aria.c
+++ b/cipher/aria.c
@@ -1656,7 +1656,7 @@ aria_setkey(void *c, const byte *key, unsigned keylen,
bulk_ops->cbc_dec = _gcry_aria_cbc_dec;
bulk_ops->cfb_enc = _gcry_aria_cfb_enc;
bulk_ops->cfb_dec = _gcry_aria_cfb_dec;
- bulk_ops->ctr_enc = _gcry_aria_ctr_enc;
+ bulk_ops->ctr16be_enc = _gcry_aria_ctr_enc;
bulk_ops->ctr32le_enc = _gcry_aria_ctr32le_enc;
bulk_ops->ecb_crypt = _gcry_aria_ecb_crypt;
bulk_ops->xts_crypt = _gcry_aria_xts_crypt;
--
2.53.0
More information about the Gcrypt-devel
mailing list