[PATCH 04/10] camellia: reduce CTR bulk counter carry handling to 16 bits
Jussi Kivilinna
jussi.kivilinna at iki.fi
Fri Jul 24 20:50:11 CEST 2026
* cipher/camellia-glue.c (camellia_setkey): Assign ctr16be_enc bulk op.
* cipher/camellia-aesni-avx-amd64.S (inc_le128): Remove.
(_gcry_camellia_aesni_avx_ctr_enc): Add to low 16 counter bits only,
drop full-width overflow path.
* cipher/camellia-aesni-avx2-amd64.h: Likewise.
* cipher/camellia-gfni-avx512-amd64.S: Likewise.
--
Camellia CTR keystream generators did full 128-bit little-endian counter
increment with separate 64-bit overflow path. Under ctr16be_enc contract
generic ctr code splits work at low 16-bit overflow, so bulk function
adds only to low 16 counter bits. Drop inc_le128/add_le128 and overflow
path, use 16-bit stepping.
Moves camellia off transitional ctr_enc alias.
Signed-off-by: Jussi Kivilinna <jussi.kivilinna at iki.fi>
---
cipher/camellia-aesni-avx-amd64.S | 62 +++++--------
cipher/camellia-aesni-avx2-amd64.h | 129 +++++-----------------------
cipher/camellia-gfni-avx512-amd64.S | 101 +++++-----------------
cipher/camellia-glue.c | 2 +-
4 files changed, 64 insertions(+), 230 deletions(-)
diff --git a/cipher/camellia-aesni-avx-amd64.S b/cipher/camellia-aesni-avx-amd64.S
index 25354195..a29b9023 100644
--- a/cipher/camellia-aesni-avx-amd64.S
+++ b/cipher/camellia-aesni-avx-amd64.S
@@ -878,12 +878,6 @@ __camellia_dec_blk16:
CFI_ENDPROC();
ELF(.size __camellia_dec_blk16,.-__camellia_dec_blk16;)
-#define inc_le128(x, minus_one, tmp) \
- vpcmpeqq minus_one, x, tmp; \
- vpsubq minus_one, x, x; \
- vpslldq $8, tmp, tmp; \
- vpsubq tmp, x, x;
-
.align 16
.globl _gcry_camellia_aesni_avx_ctr_enc
ELF(.type _gcry_camellia_aesni_avx_ctr_enc, at function;)
@@ -924,48 +918,48 @@ _gcry_camellia_aesni_avx_ctr_enc:
vpshufb %xmm14, %xmm15, %xmm0; /* be => le */
vpcmpeqd %xmm15, %xmm15, %xmm15;
- vpsrldq $8, %xmm15, %xmm15; /* low: -1, high: 0 */
+ vpsrldq $14, %xmm15, %xmm15; /* low: -1, high: 0 */
/* construct IVs */
- inc_le128(%xmm0, %xmm15, %xmm13);
+ vpsubw %xmm15, %xmm0, %xmm0;
vpshufb %xmm14, %xmm0, %xmm13;
vmovdqu %xmm13, 14 * 16(%rax);
- inc_le128(%xmm0, %xmm15, %xmm13);
+ vpsubw %xmm15, %xmm0, %xmm0;
vpshufb %xmm14, %xmm0, %xmm13;
vmovdqu %xmm13, 13 * 16(%rax);
- inc_le128(%xmm0, %xmm15, %xmm13);
+ vpsubw %xmm15, %xmm0, %xmm0;
vpshufb %xmm14, %xmm0, %xmm12;
- inc_le128(%xmm0, %xmm15, %xmm13);
+ vpsubw %xmm15, %xmm0, %xmm0;
vpshufb %xmm14, %xmm0, %xmm11;
- inc_le128(%xmm0, %xmm15, %xmm13);
+ vpsubw %xmm15, %xmm0, %xmm0;
vpshufb %xmm14, %xmm0, %xmm10;
- inc_le128(%xmm0, %xmm15, %xmm13);
+ vpsubw %xmm15, %xmm0, %xmm0;
vpshufb %xmm14, %xmm0, %xmm9;
- inc_le128(%xmm0, %xmm15, %xmm13);
+ vpsubw %xmm15, %xmm0, %xmm0;
vpshufb %xmm14, %xmm0, %xmm8;
- inc_le128(%xmm0, %xmm15, %xmm13);
+ vpsubw %xmm15, %xmm0, %xmm0;
vpshufb %xmm14, %xmm0, %xmm7;
- inc_le128(%xmm0, %xmm15, %xmm13);
+ vpsubw %xmm15, %xmm0, %xmm0;
vpshufb %xmm14, %xmm0, %xmm6;
- inc_le128(%xmm0, %xmm15, %xmm13);
+ vpsubw %xmm15, %xmm0, %xmm0;
vpshufb %xmm14, %xmm0, %xmm5;
- inc_le128(%xmm0, %xmm15, %xmm13);
+ vpsubw %xmm15, %xmm0, %xmm0;
vpshufb %xmm14, %xmm0, %xmm4;
- inc_le128(%xmm0, %xmm15, %xmm13);
+ vpsubw %xmm15, %xmm0, %xmm0;
vpshufb %xmm14, %xmm0, %xmm3;
- inc_le128(%xmm0, %xmm15, %xmm13);
+ vpsubw %xmm15, %xmm0, %xmm0;
vpshufb %xmm14, %xmm0, %xmm2;
- inc_le128(%xmm0, %xmm15, %xmm13);
+ vpsubw %xmm15, %xmm0, %xmm0;
vpshufb %xmm14, %xmm0, %xmm1;
- inc_le128(%xmm0, %xmm15, %xmm13);
- vmovdqa %xmm0, %xmm13;
+ vpsubw %xmm15, %xmm0, %xmm0;
vpshufb %xmm14, %xmm0, %xmm0;
- inc_le128(%xmm13, %xmm15, %xmm14);
- vpshufb .Lbswap128_mask rRIP, %xmm13, %xmm13; /* le => be */
- vmovdqu %xmm13, (%rcx);
.align 8
.Lload_ctr_done:
+ /* Update IV */
+ addb $16, 15(%rcx);
+ adcb $0, 14(%rcx);
+
/* inpack16_pre: */
vmovq (key_table)(CTX), %xmm15;
vpshufb .Lpack_bswap rRIP, %xmm15, %xmm15;
@@ -1016,24 +1010,8 @@ _gcry_camellia_aesni_avx_ctr_enc:
ret_spec_stop;
.align 8
-.Lctr_byteadd_full_ctr_carry:
- movq 8(%rcx), %r11;
- movq (%rcx), %r10;
- bswapq %r11;
- bswapq %r10;
- addq $16, %r11;
- adcq $0, %r10;
- bswapq %r11;
- bswapq %r10;
- movq %r11, 8(%rcx);
- movq %r10, (%rcx);
- jmp .Lctr_byteadd_xmm;
-.align 8
.Lctr_byteadd:
vmovdqu (%rcx), %xmm15;
- je .Lctr_byteadd_full_ctr_carry;
- addb $16, 15(%rcx);
-.Lctr_byteadd_xmm:
vmovdqa %xmm15, %xmm0;
vpaddb .Lbige_addb_1 rRIP, %xmm15, %xmm14;
vmovdqu %xmm15, 15 * 16(%rax);
diff --git a/cipher/camellia-aesni-avx2-amd64.h b/cipher/camellia-aesni-avx2-amd64.h
index 4c3fb4b2..21933c34 100644
--- a/cipher/camellia-aesni-avx2-amd64.h
+++ b/cipher/camellia-aesni-avx2-amd64.h
@@ -1164,12 +1164,6 @@ FUNC_NAME(dec_blk32):
CFI_ENDPROC();
ELF(.size FUNC_NAME(dec_blk32),.-FUNC_NAME(dec_blk32);)
-#define inc_le128(x, minus_one, tmp) \
- vpcmpeqq minus_one, x, tmp; \
- vpsubq minus_one, x, x; \
- vpslldq $8, tmp, tmp; \
- vpsubq tmp, x, x;
-
.align 16
.globl FUNC_NAME(ctr_enc)
ELF(.type FUNC_NAME(ctr_enc), at function;)
@@ -1200,123 +1194,60 @@ FUNC_NAME(ctr_enc):
cmpb $(0x100 - 32), 15(%rcx);
jbe .Lctr_byteadd;
- movq 8(%rcx), %r11;
- bswapq %r11;
-
vpcmpeqd %ymm15, %ymm15, %ymm15;
- vpsrldq $8, %ymm15, %ymm15; /* ab: -1:0 ; cd: -1:0 */
+ vpsrldq $14, %ymm15, %ymm15; /* ab: -1:0 ; cd: -1:0 */
/* load IV and byteswap */
vmovdqu (%rcx), %xmm0;
vpshufb .Lbswap128_mask rRIP, %xmm0, %xmm0;
vmovdqa %xmm0, %xmm1;
- inc_le128(%xmm0, %xmm15, %xmm14);
+ vpsubw %xmm15, %xmm0, %xmm0;
vbroadcasti128 .Lbswap128_mask rRIP, %ymm14;
vinserti128 $1, %xmm0, %ymm1, %ymm0;
vpshufb %ymm14, %ymm0, %ymm13;
vmovdqu %ymm13, 15 * 32(%rax);
- /* check need for handling 64-bit overflow and carry */
- cmpq $(0xffffffffffffffff - 32), %r11;
- ja .Lload_ctr_carry;
-
/* construct IVs */
- vpaddq %ymm15, %ymm15, %ymm15; /* ab: -2:0 ; cd: -2:0 */
- vpsubq %ymm15, %ymm0, %ymm0;
+ vpaddw %ymm15, %ymm15, %ymm15; /* ab: -2:0 ; cd: -2:0 */
+ vpsubw %ymm15, %ymm0, %ymm0;
vpshufb %ymm14, %ymm0, %ymm13;
vmovdqu %ymm13, 14 * 32(%rax);
- vpsubq %ymm15, %ymm0, %ymm0;
+ vpsubw %ymm15, %ymm0, %ymm0;
vpshufb %ymm14, %ymm0, %ymm13;
vmovdqu %ymm13, 13 * 32(%rax);
- vpsubq %ymm15, %ymm0, %ymm0;
+ vpsubw %ymm15, %ymm0, %ymm0;
vpshufb %ymm14, %ymm0, %ymm12;
- vpsubq %ymm15, %ymm0, %ymm0;
+ vpsubw %ymm15, %ymm0, %ymm0;
vpshufb %ymm14, %ymm0, %ymm11;
- vpsubq %ymm15, %ymm0, %ymm0;
+ vpsubw %ymm15, %ymm0, %ymm0;
vpshufb %ymm14, %ymm0, %ymm10;
- vpsubq %ymm15, %ymm0, %ymm0;
+ vpsubw %ymm15, %ymm0, %ymm0;
vpshufb %ymm14, %ymm0, %ymm9;
- vpsubq %ymm15, %ymm0, %ymm0;
+ vpsubw %ymm15, %ymm0, %ymm0;
vpshufb %ymm14, %ymm0, %ymm8;
- vpsubq %ymm15, %ymm0, %ymm0;
+ vpsubw %ymm15, %ymm0, %ymm0;
vpshufb %ymm14, %ymm0, %ymm7;
- vpsubq %ymm15, %ymm0, %ymm0;
+ vpsubw %ymm15, %ymm0, %ymm0;
vpshufb %ymm14, %ymm0, %ymm6;
- vpsubq %ymm15, %ymm0, %ymm0;
+ vpsubw %ymm15, %ymm0, %ymm0;
vpshufb %ymm14, %ymm0, %ymm5;
- vpsubq %ymm15, %ymm0, %ymm0;
+ vpsubw %ymm15, %ymm0, %ymm0;
vpshufb %ymm14, %ymm0, %ymm4;
- vpsubq %ymm15, %ymm0, %ymm0;
+ vpsubw %ymm15, %ymm0, %ymm0;
vpshufb %ymm14, %ymm0, %ymm3;
- vpsubq %ymm15, %ymm0, %ymm0;
+ vpsubw %ymm15, %ymm0, %ymm0;
vpshufb %ymm14, %ymm0, %ymm2;
- vpsubq %ymm15, %ymm0, %ymm0;
+ vpsubw %ymm15, %ymm0, %ymm0;
vpshufb %ymm14, %ymm0, %ymm1;
- vpsubq %ymm15, %ymm0, %ymm0; /* +30 ; +31 */
- vpsubq %xmm15, %xmm0, %xmm13; /* +32 */
+ vpsubw %ymm15, %ymm0, %ymm0; /* +30 ; +31 */
vpshufb %ymm14, %ymm0, %ymm0;
- vpshufb %xmm14, %xmm13, %xmm13;
- vmovdqu %xmm13, (%rcx);
-
- jmp .Lload_ctr_done;
-
-.align 4
-.Lload_ctr_carry:
- /* construct IVs */
- inc_le128(%ymm0, %ymm15, %ymm13); /* ab: le1 ; cd: le2 */
- inc_le128(%ymm0, %ymm15, %ymm13); /* ab: le2 ; cd: le3 */
- vpshufb %ymm14, %ymm0, %ymm13;
- vmovdqu %ymm13, 14 * 32(%rax);
- inc_le128(%ymm0, %ymm15, %ymm13);
- inc_le128(%ymm0, %ymm15, %ymm13);
- vpshufb %ymm14, %ymm0, %ymm13;
- vmovdqu %ymm13, 13 * 32(%rax);
- inc_le128(%ymm0, %ymm15, %ymm13);
- inc_le128(%ymm0, %ymm15, %ymm13);
- vpshufb %ymm14, %ymm0, %ymm12;
- inc_le128(%ymm0, %ymm15, %ymm13);
- inc_le128(%ymm0, %ymm15, %ymm13);
- vpshufb %ymm14, %ymm0, %ymm11;
- inc_le128(%ymm0, %ymm15, %ymm13);
- inc_le128(%ymm0, %ymm15, %ymm13);
- vpshufb %ymm14, %ymm0, %ymm10;
- inc_le128(%ymm0, %ymm15, %ymm13);
- inc_le128(%ymm0, %ymm15, %ymm13);
- vpshufb %ymm14, %ymm0, %ymm9;
- inc_le128(%ymm0, %ymm15, %ymm13);
- inc_le128(%ymm0, %ymm15, %ymm13);
- vpshufb %ymm14, %ymm0, %ymm8;
- inc_le128(%ymm0, %ymm15, %ymm13);
- inc_le128(%ymm0, %ymm15, %ymm13);
- vpshufb %ymm14, %ymm0, %ymm7;
- inc_le128(%ymm0, %ymm15, %ymm13);
- inc_le128(%ymm0, %ymm15, %ymm13);
- vpshufb %ymm14, %ymm0, %ymm6;
- inc_le128(%ymm0, %ymm15, %ymm13);
- inc_le128(%ymm0, %ymm15, %ymm13);
- vpshufb %ymm14, %ymm0, %ymm5;
- inc_le128(%ymm0, %ymm15, %ymm13);
- inc_le128(%ymm0, %ymm15, %ymm13);
- vpshufb %ymm14, %ymm0, %ymm4;
- inc_le128(%ymm0, %ymm15, %ymm13);
- inc_le128(%ymm0, %ymm15, %ymm13);
- vpshufb %ymm14, %ymm0, %ymm3;
- inc_le128(%ymm0, %ymm15, %ymm13);
- inc_le128(%ymm0, %ymm15, %ymm13);
- vpshufb %ymm14, %ymm0, %ymm2;
- inc_le128(%ymm0, %ymm15, %ymm13);
- inc_le128(%ymm0, %ymm15, %ymm13);
- vpshufb %ymm14, %ymm0, %ymm1;
- inc_le128(%ymm0, %ymm15, %ymm13);
- inc_le128(%ymm0, %ymm15, %ymm13);
- vextracti128 $1, %ymm0, %xmm13;
- vpshufb %ymm14, %ymm0, %ymm0;
- inc_le128(%xmm13, %xmm15, %xmm14);
- vpshufb .Lbswap128_mask rRIP, %xmm13, %xmm13;
- vmovdqu %xmm13, (%rcx);
.align 8
.Lload_ctr_done:
+ /* Update IV */
+ addb $32, 15(%rcx);
+ adcb $0, 14(%rcx);
+
/* inpack32_pre: */
vpbroadcastq (key_table)(CTX), %ymm15;
vpshufb .Lpack_bswap rRIP, %ymm15, %ymm15;
@@ -1367,24 +1298,8 @@ FUNC_NAME(ctr_enc):
ret_spec_stop;
.align 8
-.Lctr_byteadd_full_ctr_carry:
- movq 8(%rcx), %r11;
- movq (%rcx), %r10;
- bswapq %r11;
- bswapq %r10;
- addq $32, %r11;
- adcq $0, %r10;
- bswapq %r11;
- bswapq %r10;
- movq %r11, 8(%rcx);
- movq %r10, (%rcx);
- jmp .Lctr_byteadd_ymm;
-.align 8
.Lctr_byteadd:
vbroadcasti128 (%rcx), %ymm8;
- je .Lctr_byteadd_full_ctr_carry;
- addb $32, 15(%rcx);
-.Lctr_byteadd_ymm:
vpaddb .Lbige_addb_16_16 rRIP, %ymm8, %ymm0;
vpaddb .Lbige_addb_0_1 rRIP, %ymm8, %ymm15;
vpaddb .Lbige_addb_2_3 rRIP, %ymm8, %ymm14;
diff --git a/cipher/camellia-gfni-avx512-amd64.S b/cipher/camellia-gfni-avx512-amd64.S
index 22ae43d9..4725b06a 100644
--- a/cipher/camellia-gfni-avx512-amd64.S
+++ b/cipher/camellia-gfni-avx512-amd64.S
@@ -578,8 +578,6 @@ ELF(.type _gcry_camellia_gfni_avx512__constants, at object;)
.quad 8, 0
.Lcounter16161616_lo:
.quad 16, 0
-.Lcounter1111_hi:
- .quad 0, 1
.Lshufb_16x16b:
.byte SHUFB_BYTES(0), SHUFB_BYTES(1), SHUFB_BYTES(2), SHUFB_BYTES(3)
@@ -834,12 +832,6 @@ __camellia_gfni_avx512_dec_blk64:
CFI_ENDPROC();
ELF(.size __camellia_gfni_avx512_dec_blk64,.-__camellia_gfni_avx512_dec_blk64;)
-#define add_le128(out, in, lo_counter, hi_counter1) \
- vpaddq lo_counter, in, out; \
- vpcmpuq $1, lo_counter, out, %k1; \
- kaddb %k1, %k1, %k1; \
- vpaddq hi_counter1, out, out{%k1};
-
.align 16
.globl _gcry_camellia_gfni_avx512_ctr_enc
ELF(.type _gcry_camellia_gfni_avx512_ctr_enc, at function;)
@@ -867,69 +859,34 @@ _gcry_camellia_gfni_avx512_ctr_enc:
vbroadcasti64x2 .Lcounter4444_lo rRIP, %zmm22;
vbroadcasti64x2 .Lcounter8888_lo rRIP, %zmm23;
vbroadcasti64x2 .Lcounter16161616_lo rRIP, %zmm24;
- vbroadcasti64x2 .Lcounter1111_hi rRIP, %zmm25;
/* load IV and byteswap */
- movq 8(%rcx), %r11;
- movq (%rcx), %r10;
- bswapq %r11;
- bswapq %r10;
vbroadcasti64x2 (%rcx), %zmm0;
vpshufb %zmm19, %zmm0, %zmm0;
- /* check need for handling 64-bit overflow and carry */
- cmpq $(0xffffffffffffffff - 64), %r11;
- ja .Lload_ctr_carry;
-
/* construct IVs */
- vpaddq %zmm21, %zmm0, %zmm15; /* +0:+1:+2:+3 */
- vpaddq %zmm22, %zmm15, %zmm14; /* +4:+5:+6:+7 */
- vpaddq %zmm23, %zmm15, %zmm13; /* +8:+9:+10:+11 */
- vpaddq %zmm23, %zmm14, %zmm12; /* +12:+13:+14:+15 */
- vpaddq %zmm24, %zmm15, %zmm11; /* +16... */
- vpaddq %zmm24, %zmm14, %zmm10; /* +20... */
- vpaddq %zmm24, %zmm13, %zmm9; /* +24... */
- vpaddq %zmm24, %zmm12, %zmm8; /* +28... */
- vpaddq %zmm24, %zmm11, %zmm7; /* +32... */
- vpaddq %zmm24, %zmm10, %zmm6; /* +36... */
- vpaddq %zmm24, %zmm9, %zmm5; /* +40... */
- vpaddq %zmm24, %zmm8, %zmm4; /* +44... */
- vpaddq %zmm24, %zmm7, %zmm3; /* +48... */
- vpaddq %zmm24, %zmm6, %zmm2; /* +52... */
- vpaddq %zmm24, %zmm5, %zmm1; /* +56... */
- vpaddq %zmm24, %zmm4, %zmm0; /* +60... */
- jmp .Lload_ctr_done;
-
-.align 4
-.Lload_ctr_carry:
- /* construct IVs */
- add_le128(%zmm15, %zmm0, %zmm21, %zmm25); /* +0:+1:+2:+3 */
- add_le128(%zmm14, %zmm15, %zmm22, %zmm25); /* +4:+5:+6:+7 */
- add_le128(%zmm13, %zmm15, %zmm23, %zmm25); /* +8:+9:+10:+11 */
- add_le128(%zmm12, %zmm14, %zmm23, %zmm25); /* +12:+13:+14:+15 */
- add_le128(%zmm11, %zmm15, %zmm24, %zmm25); /* +16... */
- add_le128(%zmm10, %zmm14, %zmm24, %zmm25); /* +20... */
- add_le128(%zmm9, %zmm13, %zmm24, %zmm25); /* +24... */
- add_le128(%zmm8, %zmm12, %zmm24, %zmm25); /* +28... */
- add_le128(%zmm7, %zmm11, %zmm24, %zmm25); /* +32... */
- add_le128(%zmm6, %zmm10, %zmm24, %zmm25); /* +36... */
- add_le128(%zmm5, %zmm9, %zmm24, %zmm25); /* +40... */
- add_le128(%zmm4, %zmm8, %zmm24, %zmm25); /* +44... */
- add_le128(%zmm3, %zmm7, %zmm24, %zmm25); /* +48... */
- add_le128(%zmm2, %zmm6, %zmm24, %zmm25); /* +52... */
- add_le128(%zmm1, %zmm5, %zmm24, %zmm25); /* +56... */
- add_le128(%zmm0, %zmm4, %zmm24, %zmm25); /* +60... */
- kxorq %k1, %k1, %k1;
-
-.align 4
-.Lload_ctr_done:
+ vpaddw %zmm21, %zmm0, %zmm15; /* +0:+1:+2:+3 */
+ vpaddw %zmm22, %zmm15, %zmm14; /* +4:+5:+6:+7 */
+ vpaddw %zmm23, %zmm15, %zmm13; /* +8:+9:+10:+11 */
+ vpaddw %zmm23, %zmm14, %zmm12; /* +12:+13:+14:+15 */
+ vpaddw %zmm24, %zmm15, %zmm11; /* +16... */
+ vpaddw %zmm24, %zmm14, %zmm10; /* +20... */
+ vpaddw %zmm24, %zmm13, %zmm9; /* +24... */
+ vpaddw %zmm24, %zmm12, %zmm8; /* +28... */
+ vpaddw %zmm24, %zmm11, %zmm7; /* +32... */
+ vpaddw %zmm24, %zmm10, %zmm6; /* +36... */
+ vpaddw %zmm24, %zmm9, %zmm5; /* +40... */
+ vpaddw %zmm24, %zmm8, %zmm4; /* +44... */
+ vpaddw %zmm24, %zmm7, %zmm3; /* +48... */
+ vpaddw %zmm24, %zmm6, %zmm2; /* +52... */
+ vpaddw %zmm24, %zmm5, %zmm1; /* +56... */
+ vpaddw %zmm24, %zmm4, %zmm0; /* +60... */
+
vbroadcasti64x2 .Lpack_bswap rRIP, %zmm17;
vpbroadcastq (key_table)(CTX), %zmm16;
vpshufb %zmm17, %zmm16, %zmm16;
/* Byte-swap IVs and update counter. */
- addq $64, %r11;
- adcq $0, %r10;
vpshufb %zmm19, %zmm15, %zmm15;
vpshufb %zmm19, %zmm14, %zmm14;
vpshufb %zmm19, %zmm13, %zmm13;
@@ -938,8 +895,6 @@ _gcry_camellia_gfni_avx512_ctr_enc:
vpshufb %zmm19, %zmm10, %zmm10;
vpshufb %zmm19, %zmm9, %zmm9;
vpshufb %zmm19, %zmm8, %zmm8;
- bswapq %r11;
- bswapq %r10;
vpshufb %zmm19, %zmm7, %zmm7;
vpshufb %zmm19, %zmm6, %zmm6;
vpshufb %zmm19, %zmm5, %zmm5;
@@ -948,11 +903,13 @@ _gcry_camellia_gfni_avx512_ctr_enc:
vpshufb %zmm19, %zmm2, %zmm2;
vpshufb %zmm19, %zmm1, %zmm1;
vpshufb %zmm19, %zmm0, %zmm0;
- movq %r11, 8(%rcx);
- movq %r10, (%rcx);
.align 16
.Lctr_inpack64_pre:
+ /* Update IV */
+ addb $64, 15(%rcx);
+ adcb $0, 14(%rcx);
+
/* inpack64_pre: */
vpxorq %zmm0, %zmm16, %zmm0;
vpxorq %zmm1, %zmm16, %zmm1;
@@ -999,24 +956,8 @@ _gcry_camellia_gfni_avx512_ctr_enc:
ret_spec_stop;
.align 16
-.Lctr_byteadd_full_ctr_carry:
- movq 8(%rcx), %r11;
- movq (%rcx), %r10;
- bswapq %r11;
- bswapq %r10;
- addq $64, %r11;
- adcq $0, %r10;
- bswapq %r11;
- bswapq %r10;
- movq %r11, 8(%rcx);
- movq %r10, (%rcx);
- jmp .Lctr_byteadd_zmm;
-.align 16
.Lctr_byteadd:
vbroadcasti64x2 (%rcx), %zmm12;
- je .Lctr_byteadd_full_ctr_carry;
- addb $64, 15(%rcx);
-.Lctr_byteadd_zmm:
vbroadcasti64x2 .Lbige_addb_16 rRIP, %zmm16;
vmovdqa64 .Lbige_addb_0_1 rRIP, %zmm17;
vmovdqa64 .Lbige_addb_4_5 rRIP, %zmm18;
diff --git a/cipher/camellia-glue.c b/cipher/camellia-glue.c
index 78ff22b9..b656d964 100644
--- a/cipher/camellia-glue.c
+++ b/cipher/camellia-glue.c
@@ -611,7 +611,7 @@ camellia_setkey(void *c, const byte *key, unsigned keylen,
memset (bulk_ops, 0, sizeof(*bulk_ops));
bulk_ops->cbc_dec = _gcry_camellia_cbc_dec;
bulk_ops->cfb_dec = _gcry_camellia_cfb_dec;
- bulk_ops->ctr_enc = _gcry_camellia_ctr_enc;
+ bulk_ops->ctr16be_enc = _gcry_camellia_ctr_enc;
bulk_ops->ocb_crypt = _gcry_camellia_ocb_crypt;
bulk_ops->ocb_auth = _gcry_camellia_ocb_auth;
bulk_ops->xts_crypt = _gcry_camellia_xts_crypt;
--
2.53.0
More information about the Gcrypt-devel
mailing list