[PATCH 01/10] cipher: reduce CTR bulk counter carry handling to 16 bits
Jussi Kivilinna
jussi.kivilinna at iki.fi
Fri Jul 24 20:50:08 CEST 2026
* cipher/cipher-internal.h (cipher_bulk_ops_t): Rename 'ctr_enc' member
to 'ctr16be_enc'.
(ctr_enc): New transitional alias macro.
(cipher_block_add): Widen 'add' to u64.
(cipher_block_add_be16): New.
* cipher/cipher-ctr.c (_gcry_cipher_ctr_encrypt_ctx): Split bulk work at
low 16-bit counter overflow and do full-width carry addition here.
* cipher/bulkhelp.h (bulk_ctr_enc_128): Use cipher_block_add_be16.
* cipher/rijndael.c (do_setkey): Assign ctr16be_enc bulk op.
(_gcry_aes_ctr_enc): Use cipher_block_add_be16.
* cipher/rijndael-aesni.c (do_aesni_ctr, do_aesni_ctr_4)
(do_aesni_ctr_8): Drop full-width counter carry handling.
* cipher/rijndael-ssse3-amd64.c (_gcry_aes_ssse3_ctr_enc): Likewise.
* cipher/rijndael-vp-simd128.h (FUNC_CTR_ENC): Likewise.
* cipher/rijndael-armv8-aarch32-ce.S (_gcry_aes_ctr_enc_armv8_ce):
Likewise.
* cipher/rijndael-armv8-aarch64-ce.S (_gcry_aes_ctr_enc_armv8_ce):
Likewise.
* cipher/rijndael-ppc-functions.h (CTR_ENC_FUNC): Likewise.
* cipher/rijndael-riscv-zvkned.c (_gcry_aes_riscv_zvkned_ctr_enc):
Likewise.
* cipher/rijndael-vaes-avx2-amd64.S (_gcry_vaes_avx2_ctr_enc_amd64):
Likewise.
* cipher/rijndael-vaes-avx2-i386.S (_gcry_vaes_avx2_ctr_enc_i386):
Likewise.
* cipher/rijndael-vaes-avx512-amd64.S (_gcry_vaes_avx512_ctr_enc_amd64):
Likewise.
* tests/basic.c (cipher_cbc_bulk_test, cipher_cfb_bulk_test)
(cipher_ctr_bulk_test): Add verbose output.
(cipher_ctr16_overflow_test, check_ctr16_overflow): New.
(check_cipher_modes): Call check_ctr16_overflow.
--
Every SIMD CTR implementation carried code to propagate big-endian
counter carry across bytes, up to full 128 bits. Move carry out of
assembly: bulk functions now add only to low 16 counter bits, and
generic ctr code splits work at each 16-bit overflow to do full-width
carry addition in C. Low 16-bit overflow never happens inside bulk
call, so per-implementation carry handling drops to low-halfword
increment.
ctr_enc bulk op renamed to ctr16be_enc. Temporary ctr_enc alias keeps
not-yet-converted ciphers building. Rijndael converted as example.
New test drives single CTR call over 0x10000+ blocks across several
start counters, checking keystream and counter state against ECB
reference.
Signed-off-by: Jussi Kivilinna <jussi.kivilinna at iki.fi>
---
cipher/bulkhelp.h | 4 +-
cipher/cipher-ctr.c | 50 ++++++++-
cipher/cipher-internal.h | 26 ++++-
cipher/rijndael-aesni.c | 123 ++++-----------------
cipher/rijndael-armv8-aarch32-ce.S | 104 +++++-------------
cipher/rijndael-armv8-aarch64-ce.S | 82 +++++++-------
cipher/rijndael-ppc-common.h | 10 ++
cipher/rijndael-ppc-functions.h | 33 +++---
cipher/rijndael-riscv-zvkned.c | 103 +++++++-----------
cipher/rijndael-ssse3-amd64.c | 13 +--
cipher/rijndael-vaes-avx2-amd64.S | 130 ++++++++--------------
cipher/rijndael-vaes-avx2-i386.S | 161 +++++++++-------------------
cipher/rijndael-vaes-avx512-amd64.S | 89 +++++----------
cipher/rijndael-vp-simd128.h | 56 ++--------
cipher/rijndael.c | 24 ++---
tests/basic.c | 154 ++++++++++++++++++++++++++
16 files changed, 524 insertions(+), 638 deletions(-)
diff --git a/cipher/bulkhelp.h b/cipher/bulkhelp.h
index 833262e2..1834a9fb 100644
--- a/cipher/bulkhelp.h
+++ b/cipher/bulkhelp.h
@@ -155,9 +155,9 @@ bulk_ctr_enc_128 (void *priv, bulk_crypt_fn_t crypt_fn, byte *outbuf,
for (i = 1; i < curr_blks; i++)
{
cipher_block_cpy (&tmpbuf[i * 16], ctr, 16);
- cipher_block_add (&tmpbuf[i * 16], i, 16);
+ cipher_block_add_be16 (&tmpbuf[i * 16], i, 16);
}
- cipher_block_add (ctr, curr_blks, 16);
+ cipher_block_add_be16 (ctr, curr_blks, 16);
nburn = crypt_fn (priv, tmpbuf, tmpbuf, curr_blks);
burn_depth = nburn > burn_depth ? nburn : burn_depth;
diff --git a/cipher/cipher-ctr.c b/cipher/cipher-ctr.c
index 98363334..1ddaa7e6 100644
--- a/cipher/cipher-ctr.c
+++ b/cipher/cipher-ctr.c
@@ -64,12 +64,52 @@ _gcry_cipher_ctr_encrypt_ctx (gcry_cipher_hd_t c,
/* Use a bulk method if available. */
nblocks = inbuflen >> blocksize_shift;
- if (nblocks && c->bulk.ctr_enc)
+ if (nblocks && c->bulk.ctr16be_enc)
{
- c->bulk.ctr_enc (algo_context, c->u_ctr.ctr, outbuf, inbuf, nblocks);
- inbuf += nblocks << blocksize_shift;
- outbuf += nblocks << blocksize_shift;
- inbuflen -= nblocks << blocksize_shift;
+ byte ctr_copy[MAX_BLOCKSIZE];
+
+ do
+ {
+ /* Bulk CTR function only handles 16-bit big-endian addition of
+ * counter as optimization. Let bulk function to process only up to
+ * next 16-bit overflow. */
+ u32 ctr_low32 = buf_get_be32(&c->u_ctr.ctr[blocksize - sizeof(u32)]);
+ unsigned int ctr_low16 = ctr_low32 & 0xffffU;
+ size_t blks_to_overflow = (size_t)0x10000U - ctr_low16;
+ size_t curr_blks = nblocks;
+ int ctr_overflows = 0;
+
+ if (nblocks >= blks_to_overflow)
+ {
+ curr_blks = blks_to_overflow;
+
+ ctr_overflows = 1;
+ cipher_block_cpy(ctr_copy, c->u_ctr.ctr, blocksize);
+ }
+
+ c->bulk.ctr16be_enc (algo_context, c->u_ctr.ctr, outbuf, inbuf,
+ curr_blks);
+
+ if (ctr_overflows)
+ {
+ /* Lower 16-bits of CTR should now be zero. */
+ ctr_low32 = buf_get_be32(&c->u_ctr.ctr[blocksize - sizeof(u32)]);
+ ctr_low16 = ctr_low32 & 0xffffU;
+ gcry_assert(ctr_low16 == 0);
+
+ /* Handle full blocksize addition. */
+ cipher_block_cpy(c->u_ctr.ctr, ctr_copy, blocksize);
+ cipher_block_add(c->u_ctr.ctr, curr_blks, blocksize);
+
+ wipememory(ctr_copy, sizeof(ctr_copy));
+ }
+
+ inbuf += curr_blks << blocksize_shift;
+ outbuf += curr_blks << blocksize_shift;
+ inbuflen -= curr_blks << blocksize_shift;
+ nblocks -= curr_blks;
+ }
+ while (nblocks);
}
/* If we don't have a bulk method use the standard method. We also
diff --git a/cipher/cipher-internal.h b/cipher/cipher-internal.h
index bc023c76..c99bd189 100644
--- a/cipher/cipher-internal.h
+++ b/cipher/cipher-internal.h
@@ -196,8 +196,8 @@ typedef struct cipher_bulk_ops
const void *inbuf_arg, size_t nblocks);
void (*ofb_enc)(void *context, unsigned char *iv, void *outbuf_arg,
const void *inbuf_arg, size_t nblocks);
- void (*ctr_enc)(void *context, unsigned char *iv, void *outbuf_arg,
- const void *inbuf_arg, size_t nblocks);
+ void (*ctr16be_enc)(void *context, unsigned char *iv, void *outbuf_arg,
+ const void *inbuf_arg, size_t nblocks);
void (*ctr32le_enc)(void *context, unsigned char *iv, void *outbuf_arg,
const void *inbuf_arg, size_t nblocks);
size_t (*ocb_crypt)(gcry_cipher_hd_t c, void *outbuf_arg,
@@ -209,6 +209,11 @@ typedef struct cipher_bulk_ops
const void *inbuf_arg, size_t nblocks, int encrypt);
} cipher_bulk_ops_t;
+/* Temporary alias for transition period from full 128-bit big-endian
+ * counter addition in bulk processing function to 16-bit big-endian
+ * addition. */
+#define ctr_enc ctr16be_enc
+
/* A VIA processor with the Padlock engine as well as the Intel AES_NI
instructions require an alignment of most data on a 16 byte
@@ -820,9 +825,9 @@ cipher_bytecounter_add (u32 ctr[2], size_t add)
}
-/* Optimized function for adding value to cipher block. */
+/* Optimized function for block-size big-endian addition. */
static inline void
-cipher_block_add(void *_dstsrc, unsigned int add, size_t blocksize)
+cipher_block_add(void *_dstsrc, u64 add, size_t blocksize)
{
byte *dstsrc = _dstsrc;
u64 s[2];
@@ -843,6 +848,19 @@ cipher_block_add(void *_dstsrc, unsigned int add, size_t blocksize)
}
+/* Optimized function for 16-bit big-endian addition to cipher block. */
+static inline void
+cipher_block_add_be16(void *_dstsrc, unsigned int add, size_t blocksize)
+{
+ byte *dstsrc = _dstsrc;
+ byte add_lo = add & 0xff;
+ byte add_hi = add >> 8;
+
+ dstsrc[blocksize - 1] += add_lo;
+ dstsrc[blocksize - 2] += add_hi + (dstsrc[blocksize - 1] < add_lo);
+}
+
+
/* Optimized function for cipher block copying */
static inline void
cipher_block_cpy(void *_dst, const void *_src, size_t blocksize)
diff --git a/cipher/rijndael-aesni.c b/cipher/rijndael-aesni.c
index dea78fd0..0093aaec 100644
--- a/cipher/rijndael-aesni.c
+++ b/cipher/rijndael-aesni.c
@@ -1073,26 +1073,10 @@ do_aesni_ctr (const RIJNDAEL_context *ctx,
#define aesenc_xmm1_xmm0 ".byte 0x66, 0x0f, 0x38, 0xdc, 0xc1\n\t"
#define aesenclast_xmm1_xmm0 ".byte 0x66, 0x0f, 0x38, 0xdd, 0xc1\n\t"
- asm volatile ("movdqa %%xmm5, %%xmm0\n\t" /* xmm0 := CTR (xmm5) */
- "pcmpeqd %%xmm1, %%xmm1\n\t"
- "psrldq $8, %%xmm1\n\t" /* xmm1 = -1 */
-
- "pshufb %%xmm6, %%xmm5\n\t"
- "psubq %%xmm1, %%xmm5\n\t" /* xmm5++ (big endian) */
-
- /* detect if 64-bit carry handling is needed */
- "cmpl $0xffffffff, 8(%[ctr])\n\t"
- "jne .Lno_carry%=\n\t"
- "cmpl $0xffffffff, 12(%[ctr])\n\t"
- "jne .Lno_carry%=\n\t"
-
- "pslldq $8, %%xmm1\n\t" /* move lower 64-bit to high */
- "psubq %%xmm1, %%xmm5\n\t" /* add carry to upper 64bits */
+ asm volatile ("movdqa (%[ctr]), %%xmm0\n\t" /* xmm0 := CTR (mem) */
- ".Lno_carry%=:\n\t"
-
- "pshufb %%xmm6, %%xmm5\n\t"
- "movdqa %%xmm5, (%[ctr])\n\t" /* Update CTR (mem). */
+ "addb $1, 15(%[ctr])\n\t" /* Update CTR (mem). */
+ "adcb $0, 14(%[ctr])\n\t"
"pxor (%[key]), %%xmm0\n\t" /* xmm1 ^= key[0] */
"movdqa 0x10(%[key]), %%xmm1\n\t"
@@ -1194,45 +1178,21 @@ do_aesni_ctr_4 (const RIJNDAEL_context *ctx,
"jmp .Ldone_ctr%=\n\t"
".Ladd32bit%=:\n\t"
- "movdqa %%xmm5, (%[ctr])\n\t" /* Restore CTR. */
+ "addb $1, 14(%[ctr])\n\t" /* Update CTR. */
"movdqa %%xmm5, %%xmm0\n\t" /* xmm0, xmm2 := CTR (xmm5) */
"movdqa %%xmm0, %%xmm2\n\t"
"pcmpeqd %%xmm1, %%xmm1\n\t"
- "psrldq $8, %%xmm1\n\t" /* xmm1 = -1 */
+ "psrldq $14, %%xmm1\n\t" /* xmm1 = -1 */
"pshufb %%xmm6, %%xmm2\n\t" /* xmm2 := le(xmm2) */
- "psubq %%xmm1, %%xmm2\n\t" /* xmm2++ */
+ "psubw %%xmm1, %%xmm2\n\t" /* xmm2++ */
"movdqa %%xmm2, %%xmm3\n\t" /* xmm3 := xmm2 */
- "psubq %%xmm1, %%xmm3\n\t" /* xmm3++ */
+ "psubw %%xmm1, %%xmm3\n\t" /* xmm3++ */
"movdqa %%xmm3, %%xmm4\n\t" /* xmm4 := xmm3 */
- "psubq %%xmm1, %%xmm4\n\t" /* xmm4++ */
+ "psubw %%xmm1, %%xmm4\n\t" /* xmm4++ */
"movdqa %%xmm4, %%xmm5\n\t" /* xmm5 := xmm4 */
- "psubq %%xmm1, %%xmm5\n\t" /* xmm5++ */
-
- /* detect if 64-bit carry handling is needed */
- "cmpl $0xffffffff, 8(%[ctr])\n\t"
- "jne .Lno_carry%=\n\t"
- "movl 12(%[ctr]), %%esi\n\t"
- "bswapl %%esi\n\t"
- "cmpl $0xfffffffc, %%esi\n\t"
- "jb .Lno_carry%=\n\t" /* no carry */
-
- "pslldq $8, %%xmm1\n\t" /* move lower 64-bit to high */
- "je .Lcarry_xmm5%=\n\t" /* esi == 0xfffffffc */
- "cmpl $0xfffffffe, %%esi\n\t"
- "jb .Lcarry_xmm4%=\n\t" /* esi == 0xfffffffd */
- "je .Lcarry_xmm3%=\n\t" /* esi == 0xfffffffe */
- /* esi == 0xffffffff */
-
- "psubq %%xmm1, %%xmm2\n\t"
- ".Lcarry_xmm3%=:\n\t"
- "psubq %%xmm1, %%xmm3\n\t"
- ".Lcarry_xmm4%=:\n\t"
- "psubq %%xmm1, %%xmm4\n\t"
- ".Lcarry_xmm5%=:\n\t"
- "psubq %%xmm1, %%xmm5\n\t"
-
- ".Lno_carry%=:\n\t"
+ "psubw %%xmm1, %%xmm5\n\t" /* xmm5++ */
+
"movdqa (%[key]), %%xmm1\n\t" /* xmm1 := key[0] */
"pshufb %%xmm6, %%xmm2\n\t" /* xmm2 := be(xmm2) */
@@ -1240,8 +1200,6 @@ do_aesni_ctr_4 (const RIJNDAEL_context *ctx,
"pshufb %%xmm6, %%xmm4\n\t" /* xmm4 := be(xmm4) */
"pshufb %%xmm6, %%xmm5\n\t" /* xmm5 := be(xmm5) */
- "movdqa %%xmm5, (%[ctr])\n\t" /* Update CTR (mem). */
-
".Ldone_ctr%=:\n\t"
:
: [ctr] "r" (ctr),
@@ -1447,67 +1405,29 @@ do_aesni_ctr_8 (const RIJNDAEL_context *ctx,
"jmp .Ldone_ctr%=\n\t"
".Ladd32bit%=:\n\t"
- "movdqa %%xmm5, (%[ctr])\n\t" /* Restore CTR. */
+ "addb $1, 14(%[ctr])\n\t" /* Update CTR. */
"movdqa %%xmm5, %%xmm0\n\t" /* xmm0, xmm2 := CTR (xmm5) */
"movdqa %%xmm0, %%xmm2\n\t"
"pcmpeqd %%xmm1, %%xmm1\n\t"
- "psrldq $8, %%xmm1\n\t" /* xmm1 = -1 */
+ "psrldq $14, %%xmm1\n\t" /* xmm1 = -1 */
"pshufb %%xmm6, %%xmm2\n\t" /* xmm2 := le(xmm2) */
- "psubq %%xmm1, %%xmm2\n\t" /* xmm2++ */
+ "psubw %%xmm1, %%xmm2\n\t" /* xmm2++ */
"movdqa %%xmm2, %%xmm3\n\t" /* xmm3 := xmm2 */
- "psubq %%xmm1, %%xmm3\n\t" /* xmm3++ */
+ "psubw %%xmm1, %%xmm3\n\t" /* xmm3++ */
"movdqa %%xmm3, %%xmm4\n\t" /* xmm4 := xmm3 */
- "psubq %%xmm1, %%xmm4\n\t" /* xmm4++ */
+ "psubw %%xmm1, %%xmm4\n\t" /* xmm4++ */
"movdqa %%xmm4, %%xmm8\n\t" /* xmm8 := xmm4 */
- "psubq %%xmm1, %%xmm8\n\t" /* xmm8++ */
+ "psubw %%xmm1, %%xmm8\n\t" /* xmm8++ */
"movdqa %%xmm8, %%xmm9\n\t" /* xmm9 := xmm8 */
- "psubq %%xmm1, %%xmm9\n\t" /* xmm9++ */
+ "psubw %%xmm1, %%xmm9\n\t" /* xmm9++ */
"movdqa %%xmm9, %%xmm10\n\t" /* xmm10 := xmm9 */
- "psubq %%xmm1, %%xmm10\n\t" /* xmm10++ */
+ "psubw %%xmm1, %%xmm10\n\t" /* xmm10++ */
"movdqa %%xmm10, %%xmm11\n\t" /* xmm11 := xmm10 */
- "psubq %%xmm1, %%xmm11\n\t" /* xmm11++ */
+ "psubw %%xmm1, %%xmm11\n\t" /* xmm11++ */
"movdqa %%xmm11, %%xmm5\n\t" /* xmm5 := xmm11 */
- "psubq %%xmm1, %%xmm5\n\t" /* xmm5++ */
-
- /* detect if 64-bit carry handling is needed */
- "cmpl $0xffffffff, 8(%[ctr])\n\t"
- "jne .Lno_carry%=\n\t"
- "movl 12(%[ctr]), %%esi\n\t"
- "bswapl %%esi\n\t"
- "cmpl $0xfffffff8, %%esi\n\t"
- "jb .Lno_carry%=\n\t" /* no carry */
-
- "pslldq $8, %%xmm1\n\t" /* move lower 64-bit to high */
- "je .Lcarry_xmm5%=\n\t" /* esi == 0xfffffff8 */
- "cmpl $0xfffffffa, %%esi\n\t"
- "jb .Lcarry_xmm11%=\n\t" /* esi == 0xfffffff9 */
- "je .Lcarry_xmm10%=\n\t" /* esi == 0xfffffffa */
- "cmpl $0xfffffffc, %%esi\n\t"
- "jb .Lcarry_xmm9%=\n\t" /* esi == 0xfffffffb */
- "je .Lcarry_xmm8%=\n\t" /* esi == 0xfffffffc */
- "cmpl $0xfffffffe, %%esi\n\t"
- "jb .Lcarry_xmm4%=\n\t" /* esi == 0xfffffffd */
- "je .Lcarry_xmm3%=\n\t" /* esi == 0xfffffffe */
- /* esi == 0xffffffff */
-
- "psubq %%xmm1, %%xmm2\n\t"
- ".Lcarry_xmm3%=:\n\t"
- "psubq %%xmm1, %%xmm3\n\t"
- ".Lcarry_xmm4%=:\n\t"
- "psubq %%xmm1, %%xmm4\n\t"
- ".Lcarry_xmm8%=:\n\t"
- "psubq %%xmm1, %%xmm8\n\t"
- ".Lcarry_xmm9%=:\n\t"
- "psubq %%xmm1, %%xmm9\n\t"
- ".Lcarry_xmm10%=:\n\t"
- "psubq %%xmm1, %%xmm10\n\t"
- ".Lcarry_xmm11%=:\n\t"
- "psubq %%xmm1, %%xmm11\n\t"
- ".Lcarry_xmm5%=:\n\t"
- "psubq %%xmm1, %%xmm5\n\t"
-
- ".Lno_carry%=:\n\t"
+ "psubw %%xmm1, %%xmm5\n\t" /* xmm5++ */
+
"movdqa (%[key]), %%xmm1\n\t" /* xmm1 := key[0] */
"movdqa 16(%[key]), %%xmm7\n\t" /* xmm7 := key[1] */
@@ -1536,7 +1456,6 @@ do_aesni_ctr_8 (const RIJNDAEL_context *ctx,
"aesenc %%xmm7, %%xmm11\n\t"
"pshufb %%xmm6, %%xmm5\n\t" /* xmm5 := be(xmm5) */
- "movdqa %%xmm5, (%[ctr])\n\t" /* Update CTR (mem). */
".align 16\n\t"
".Ldone_ctr%=:\n\t"
diff --git a/cipher/rijndael-armv8-aarch32-ce.S b/cipher/rijndael-armv8-aarch32-ce.S
index 7d85aff9..7a0623cf 100644
--- a/cipher/rijndael-armv8-aarch32-ce.S
+++ b/cipher/rijndael-armv8-aarch32-ce.S
@@ -1007,19 +1007,18 @@ _gcry_aes_ctr_enc_armv8_ce:
*/
vpush {q4-q7}
- push {r4-r12,lr} /* 4*16 + 4*10 = 104b */
- ldr r4, [sp, #(104+0)]
- ldr r5, [sp, #(104+4)]
+ push {r4-r7,lr} /* 4*16 + 4*5 = 84b */
+ ldr r4, [sp, #(84+0)]
+ ldr r5, [sp, #(84+4)]
cmp r4, #0
beq .Lctr_enc_skip
cmp r5, #12
- ldm r3, {r7-r10}
+ ldr r7, [r3, #12]
vld1.8 {q0}, [r3] /* load IV */
rev r7, r7
- rev r8, r8
- rev r9, r9
- rev r10, r10
+
+ mov r5, #1
aes_preload_keys(r0, r6);
@@ -1032,58 +1031,22 @@ _gcry_aes_ctr_enc_armv8_ce:
blo .Lctr_enc_loop_##bits; \
\
.Lctr_enc_loop4_##bits: \
- cmp r10, #0xfffffffc; \
+ vmov.i8 q2, #0; \
sub r4, r4, #4; \
- blo .Lctr_enc_loop4_##bits##_nocarry; \
- cmp r9, #0xffffffff; \
- bne .Lctr_enc_loop4_##bits##_nocarry; \
- \
- adds r10, #1; \
- vmov q1, q0; \
- blcs .Lctr_overflow_one; \
- rev r11, r10; \
- vmov.32 d1[1], r11; \
- \
- adds r10, #1; \
- vmov q2, q0; \
- blcs .Lctr_overflow_one; \
- rev r11, r10; \
- vmov.32 d1[1], r11; \
- \
- adds r10, #1; \
- vmov q3, q0; \
- blcs .Lctr_overflow_one; \
- rev r11, r10; \
- vmov.32 d1[1], r11; \
- \
- adds r10, #1; \
- vmov q4, q0; \
- blcs .Lctr_overflow_one; \
- rev r11, r10; \
- vmov.32 d1[1], r11; \
- \
- b .Lctr_enc_loop4_##bits##_store_ctr; \
- \
- .Lctr_enc_loop4_##bits##_nocarry: \
- \
- veor q2, q2; \
- vrev64.8 q1, q0; \
- vceq.u32 d5, d5; \
- vadd.u64 q3, q2, q2; \
- vadd.u64 q4, q3, q2; \
- vadd.u64 q0, q3, q3; \
- vsub.u64 q2, q1, q2; \
- vsub.u64 q3, q1, q3; \
- vsub.u64 q4, q1, q4; \
- vsub.u64 q0, q1, q0; \
- vrev64.8 q1, q1; \
- vrev64.8 q2, q2; \
- vrev64.8 q3, q3; \
- vrev64.8 q0, q0; \
- vrev64.8 q4, q4; \
- add r10, #4; \
- \
- .Lctr_enc_loop4_##bits##_store_ctr: \
+ vmov.16 d5[3], r5; \
+ vrev16.8 q1, q0; \
+ \
+ vadd.u16 q0, q2, q2; \
+ vadd.u16 q2, q1, q2; \
+ vadd.u16 q3, q1, q0; \
+ vrev16.8 q1, q1; \
+ vadd.u16 q4, q2, q0; \
+ vrev16.8 q2, q2; \
+ vadd.u16 q0, q3, q0; \
+ vrev16.8 q3, q3; \
+ vrev16.8 q4, q4; \
+ vrev16.8 q0, q0; \
+ add r7, #4; \
\
vst1.8 {q0}, [r3]; \
cmp r4, #4; \
@@ -1109,13 +1072,12 @@ _gcry_aes_ctr_enc_armv8_ce:
\
.Lctr_enc_loop_##bits: \
\
- adds r10, #1; \
+ add r7, #1; \
vmov q1, q0; \
- blcs .Lctr_overflow_one; \
- rev r11, r10; \
+ rev r5, r7; \
subs r4, r4, #1; \
vld1.8 {q2}, [r2]!; /* load ciphertext */ \
- vmov.32 d1[1], r11; \
+ vmov.32 d1[1], r5; \
\
do_aes_one##bits(e, mc, q1, q1, ##__VA_ARGS__); \
\
@@ -1148,21 +1110,9 @@ _gcry_aes_ctr_enc_armv8_ce:
CLEAR_REG(q15)
.Lctr_enc_skip:
- pop {r4-r12,lr}
+ pop {r4-r7,lr}
vpop {q4-q7}
bx lr
-
-.Lctr_overflow_one:
- adcs r9, #0
- adcs r8, #0
- adc r7, #0
- rev r11, r9
- rev r12, r8
- vmov.32 d1[0], r11
- rev r11, r7
- vmov.32 d0[1], r12
- vmov.32 d0[0], r11
- bx lr
.size _gcry_aes_ctr_enc_armv8_ce,.-_gcry_aes_ctr_enc_armv8_ce;
@@ -1208,7 +1158,7 @@ _gcry_aes_ctr32le_enc_armv8_ce:
blo .Lctr32le_enc_loop_##bits; \
\
.Lctr32le_enc_loop4_##bits: \
- veor q2, q2; \
+ vmov.i8 q2, #0; \
sub r4, r4, #4; \
vmov.i64 d4, #0xffffffff; /* q2 <= -1:0:0:0 */ \
vmov q1, q0; \
@@ -1244,7 +1194,7 @@ _gcry_aes_ctr32le_enc_armv8_ce:
\
.Lctr32le_enc_loop_##bits: \
\
- veor q2, q2; \
+ vmov.i8 q2, #0; \
vmov q1, q0; \
vmov.i64 d4, #0xffffffff; /* q2 <= -1:0:0:0 */ \
subs r4, r4, #1; \
diff --git a/cipher/rijndael-armv8-aarch64-ce.S b/cipher/rijndael-armv8-aarch64-ce.S
index 64f67fbe..5152fab7 100644
--- a/cipher/rijndael-armv8-aarch64-ce.S
+++ b/cipher/rijndael-armv8-aarch64-ce.S
@@ -734,14 +734,11 @@ _gcry_aes_ctr_enc_armv8_ce:
movi v16.16b, #0
mov v16.S[3], w6 /* 1 */
- /* load IV */
- ldp x9, x10, [x3]
+ /* load IV, x9 gets counter low byte at bits 56:63 for carry detection */
+ ldr x9, [x3, #8]
ld1 {v0.16b}, [x3]
- rev x9, x9
- rev x10, x10
- mov x12, #(4 << 56)
- lsl x11, x10, #56
+ mov x10, #(4 << 56)
aes_preload_keys(x0, w5);
@@ -751,25 +748,25 @@ _gcry_aes_ctr_enc_armv8_ce:
#define CTR_ENC(bits) \
.Lctr_enc_entry_##bits: \
cmp x4, #4; \
- b.lo .Lctr_enc_loop_##bits; \
+ b.lo .Lctr_enc_loop_entry_##bits; \
\
st1 {v8.16b-v11.16b}, [sp]; /* store callee saved registers */ \
\
- adds x11, x11, x12; \
- add v9.4s, v16.4s, v16.4s; /* 2 */ \
- add v10.4s, v16.4s, v9.4s; /* 3 */ \
- add v11.4s, v9.4s, v9.4s; /* 4 */ \
+ adds x9, x9, x10; \
+ add v9.16b, v16.16b, v16.16b; /* 2 */ \
+ add v10.16b, v16.16b, v9.16b; /* 3 */ \
+ add v11.16b, v9.16b, v9.16b; /* 4 */ \
mov x7, #1; \
sub x4, x4, #4; \
ld1 {v5.16b-v8.16b}, [x2], #64; /* preload ciphertext */ \
b.cs .Lctr_enc_carry4_##bits; \
\
+ /* 8-bit addition */ \
mov v1.16b, v0.16b; \
- add x10, x10, #4; \
add v2.16b, v0.16b, v16.16b; \
- add v3.8h, v0.8h, v9.8h; \
- add v4.4s, v0.4s, v10.4s; \
- add v0.2d, v0.2d, v11.2d; \
+ add v3.16b, v0.16b, v9.16b; \
+ add v4.16b, v0.16b, v10.16b; \
+ add v0.16b, v0.16b, v11.16b; \
\
.Lctr_enc_entry4_##bits##_carry_done: \
mov x7, #0; \
@@ -786,16 +783,16 @@ _gcry_aes_ctr_enc_armv8_ce:
eor v8.16b, v8.16b, vklast.16b; \
do_aes_4_part2_##bits(e, mc, v12, v13, v14, v15, v1, v2, v3, v4, v5, v6, v7, v8); \
ld1 {v5.16b-v8.16b}, [x2], #64; /* preload ciphertext */ \
- adds x11, x11, x12; \
+ adds x9, x9, x10; \
sub x4, x4, #4; \
b.cs .Lctr_enc_carry4_##bits; \
\
+ /* 8-bit addition */ \
mov v1.16b, v0.16b; \
- add x10, x10, #4; \
add v2.16b, v0.16b, v16.16b; \
- add v3.8h, v0.8h, v9.8h; \
- add v4.4s, v0.4s, v10.4s; \
- add v0.2d, v0.2d, v11.2d; \
+ add v3.16b, v0.16b, v9.16b; \
+ add v4.16b, v0.16b, v10.16b; \
+ add v0.16b, v0.16b, v11.16b; \
\
.Lctr_enc_loop4_##bits##_carry_done: \
cmp x4, #4; \
@@ -815,7 +812,6 @@ _gcry_aes_ctr_enc_armv8_ce:
\
st1 {v5.16b-v8.16b}, [x1], #64; /* store plaintext */ \
\
- CLEAR_REG(v3); \
CLEAR_REG(v4); \
ld1 {v8.16b-v11.16b}, [sp]; /* restore callee saved registers */ \
CLEAR_REG(v5); \
@@ -823,16 +819,17 @@ _gcry_aes_ctr_enc_armv8_ce:
CLEAR_REG(v7); \
cbz x4, .Lctr_enc_done; \
\
+ .Lctr_enc_loop_entry_##bits: \
+ rev16 v3.16b, v16.16b; \
+ \
.Lctr_enc_loop_##bits: \
\
- adds x10, x10, #1; \
mov v1.16b, v0.16b; \
- adc x9, x9, xzr; \
- dup v0.2d, x10; \
+ rev16 v0.16b, v0.16b; /* be->le */ \
+ add v0.8h, v0.8h, v3.8h; \
sub x4, x4, #1; \
- ins v0.D[0], x9; \
ld1 {v2.16b}, [x2], #16; /* load ciphertext */ \
- rev64 v0.16b, v0.16b; \
+ rev16 v0.16b, v0.16b; \
\
do_aes_one_part1(e, mc, v1, vk0); \
eor v2.16b, v2.16b, vklast.16b; \
@@ -846,27 +843,19 @@ _gcry_aes_ctr_enc_armv8_ce:
\
.Lctr_enc_carry4_##bits: \
\
- adds x13, x10, #1; \
+ /* 16-bit addition */ \
+ rev16 v4.16b, v0.16b; /* be->le */ \
+ rev16 v3.16b, v16.16b; \
mov v1.16b, v0.16b; \
- adc x14, x9, xzr; \
- dup v2.2d, x13; \
- adds x13, x10, #2; \
- ins v2.D[0], x14; \
- adc x14, x9, xzr; \
- rev64 v2.16b, v2.16b; \
- dup v3.2d, x13; \
- adds x13, x10, #3; \
- ins v3.D[0], x14; \
- adc x14, x9, xzr; \
- rev64 v3.16b, v3.16b; \
- dup v4.2d, x13; \
- adds x10, x10, #4; \
- ins v4.D[0], x14; \
- adc x9, x9, xzr; \
- rev64 v4.16b, v4.16b; \
- dup v0.2d, x10; \
- ins v0.D[0], x9; \
- rev64 v0.16b, v0.16b; \
+ rev16 v0.16b, v9.16b; \
+ add v2.8h, v4.8h, v3.8h; /* ctr+1 */ \
+ add v3.8h, v4.8h, v0.8h; /* ctr+2 */ \
+ add v4.8h, v0.8h, v2.8h; /* ctr+1+2 */ \
+ add v0.8h, v0.8h, v3.8h; /* ctr+2+2 */ \
+ rev16 v2.16b, v2.16b; /* le->be */ \
+ rev16 v3.16b, v3.16b; /* le->be */ \
+ rev16 v4.16b, v4.16b; /* le->be */ \
+ rev16 v0.16b, v0.16b; /* le->be */ \
\
cbz x7, .Lctr_enc_loop4_##bits##_carry_done; \
b .Lctr_enc_entry4_##bits##_carry_done;
@@ -885,6 +874,7 @@ _gcry_aes_ctr_enc_armv8_ce:
CLEAR_REG(v0)
CLEAR_REG(v1)
CLEAR_REG(v2)
+ CLEAR_REG(v3)
CLEAR_REG(v16)
add sp, sp, #128;
diff --git a/cipher/rijndael-ppc-common.h b/cipher/rijndael-ppc-common.h
index 611b5871..d7a35c26 100644
--- a/cipher/rijndael-ppc-common.h
+++ b/cipher/rijndael-ppc-common.h
@@ -232,6 +232,16 @@ asm_add_uint64(block a, block b)
return res;
}
+static ASM_FUNC_ATTR_INLINE block
+asm_add_uint16(block a, block b)
+{
+ block res;
+ __asm__ volatile ("vadduhm %0,%1,%2\n\t"
+ : "=v" (res)
+ : "v" (a), "v" (b));
+ return res;
+}
+
static ASM_FUNC_ATTR_INLINE block
asm_sra_int64(block a, block b)
{
diff --git a/cipher/rijndael-ppc-functions.h b/cipher/rijndael-ppc-functions.h
index eb39717d..31b75aee 100644
--- a/cipher/rijndael-ppc-functions.h
+++ b/cipher/rijndael-ppc-functions.h
@@ -888,25 +888,24 @@ CTR_ENC_FUNC (void *context, unsigned char *ctr_arg, void *outbuf_arg,
{
block in0, in1, in2, in3, in4, in5, in6, in7;
block b0, b1, b2, b3, b4, b5, b6, b7;
- block two, three, four;
+ block two, four;
block rkey;
- two = asm_add_uint128 (one, one);
- three = asm_add_uint128 (two, one);
- four = asm_add_uint128 (two, two);
+ two = asm_add_uint16 (one, one);
+ four = asm_add_uint16 (two, two);
for (; nblocks >= 8; nblocks -= 8)
{
- b1 = asm_add_uint128 (ctr, one);
- b2 = asm_add_uint128 (ctr, two);
- b3 = asm_add_uint128 (ctr, three);
- b4 = asm_add_uint128 (ctr, four);
- b5 = asm_add_uint128 (b1, four);
- b6 = asm_add_uint128 (b2, four);
- b7 = asm_add_uint128 (b3, four);
+ b1 = asm_add_uint16 (ctr, one);
+ b2 = asm_add_uint16 (ctr, two);
+ b3 = asm_add_uint16 (b1, two);
+ b4 = asm_add_uint16 (b2, two);
+ b5 = asm_add_uint16 (b1, four);
+ b6 = asm_add_uint16 (b2, four);
+ b7 = asm_add_uint16 (b3, four);
b0 = asm_xor (rkey0, ctr);
rkey = ALIGNED_LOAD (rk, 1);
- ctr = asm_add_uint128 (b4, four);
+ ctr = asm_add_uint16 (b4, four);
b1 = asm_xor (rkey0, b1);
b2 = asm_xor (rkey0, b2);
b3 = asm_xor (rkey0, b3);
@@ -1012,11 +1011,11 @@ CTR_ENC_FUNC (void *context, unsigned char *ctr_arg, void *outbuf_arg,
if (nblocks >= 4)
{
- b1 = asm_add_uint128 (ctr, one);
- b2 = asm_add_uint128 (ctr, two);
- b3 = asm_add_uint128 (ctr, three);
+ b1 = asm_add_uint16 (ctr, one);
+ b2 = asm_add_uint16 (ctr, two);
+ b3 = asm_add_uint16 (b1, two);
b0 = asm_xor (rkey0, ctr);
- ctr = asm_add_uint128 (ctr, four);
+ ctr = asm_add_uint16 (b2, two);
b1 = asm_xor (rkey0, b1);
b2 = asm_xor (rkey0, b2);
b3 = asm_xor (rkey0, b3);
@@ -1080,7 +1079,7 @@ CTR_ENC_FUNC (void *context, unsigned char *ctr_arg, void *outbuf_arg,
for (; nblocks; nblocks--)
{
b = ctr;
- ctr = asm_add_uint128 (ctr, one);
+ ctr = asm_add_uint16 (ctr, one);
rkeylast = rkeylast_orig ^ VEC_LOAD_BE (in, 0, bige_const);
AES_ENCRYPT (b, rounds);
diff --git a/cipher/rijndael-riscv-zvkned.c b/cipher/rijndael-riscv-zvkned.c
index 9472a99e..ed3887ae 100644
--- a/cipher/rijndael-riscv-zvkned.c
+++ b/cipher/rijndael-riscv-zvkned.c
@@ -738,8 +738,7 @@ _gcry_aes_riscv_zvkned_ctr_enc (void *context, unsigned char *ctr_arg,
{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3 },
{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4 }
};
- static const u64 carry_add[2] = { 1, 1 };
- static const u64 nocarry_add[2] = { 1, 0 };
+ static const u32 le_add[4] = { 1, 0, 0, 0 };
RIJNDAEL_context *ctx = context;
unsigned char *outbuf = outbuf_arg;
const unsigned char *inbuf = inbuf_arg;
@@ -749,12 +748,12 @@ _gcry_aes_riscv_zvkned_ctr_enc (void *context, unsigned char *ctr_arg,
size_t vl_bytes = vl * 4;
u64 ctrlow;
vuint32m1_t ctr;
- vuint8m1_t add1;
+ vuint32m1_t add1;
ROUND_KEY_VARIABLES;
PRELOAD_ROUND_KEYS (rk, rounds, vl);
- add1 = __riscv_vle8_v_u8m1(add_u8_array[0], vl_bytes);
+ add1 = cast_u8m1_u32m1(__riscv_vle8_v_u8m1(add_u8_array[0], vl_bytes));
ctr = unaligned_load_u32m1(ctr_arg, vl);
ctrlow = __riscv_vmv_x_s_u64m1_u64(cast_u32m1_u64m1(bswap128_u32m1(ctr, vl)));
@@ -762,9 +761,12 @@ _gcry_aes_riscv_zvkned_ctr_enc (void *context, unsigned char *ctr_arg,
if (nblocks >= 4)
{
- vuint8m1_t add2 = __riscv_vle8_v_u8m1(add_u8_array[1], vl_bytes);
- vuint8m1_t add3 = __riscv_vle8_v_u8m1(add_u8_array[2], vl_bytes);
- vuint8m1_t add4 = __riscv_vle8_v_u8m1(add_u8_array[3], vl_bytes);
+ vuint32m1_t add2 = cast_u8m1_u32m1(__riscv_vle8_v_u8m1(add_u8_array[1],
+ vl_bytes));
+ vuint32m1_t add3 = cast_u8m1_u32m1(__riscv_vle8_v_u8m1(add_u8_array[2],
+ vl_bytes));
+ vuint32m1_t add4 = cast_u8m1_u32m1(__riscv_vle8_v_u8m1(add_u8_array[3],
+ vl_bytes));
memory_barrier_with_vec(add2);
memory_barrier_with_vec(add3);
@@ -778,67 +780,43 @@ _gcry_aes_riscv_zvkned_ctr_enc (void *context, unsigned char *ctr_arg,
/* detect if 8-bit carry handling is needed */
if (UNLIKELY(((ctrlow += 4) & 0xff) <= 3))
{
- static const u64 *adders[5][4] =
- {
- { nocarry_add, nocarry_add, nocarry_add, carry_add },
- { nocarry_add, nocarry_add, carry_add, nocarry_add },
- { nocarry_add, carry_add, nocarry_add, nocarry_add },
- { carry_add, nocarry_add, nocarry_add, nocarry_add },
- { nocarry_add, nocarry_add, nocarry_add, nocarry_add }
- };
- unsigned int idx = ctrlow <= 3 ? ctrlow : 4;
- vuint64m1_t ctr_u64;
- vuint32m1_t ctr_u32_1;
- vuint32m1_t ctr_u32_2;
- vuint32m1_t ctr_u32_3;
- vuint32m1_t ctr_u32_4;
- vuint64m1_t add_u64;
+ vuint32m1_t add_1 = __riscv_vle32_v_u32m1(le_add, vl);
+ vuint32m1_t add_2 = __riscv_vadd_vv_u32m1(add_1, add_1, vl);
+ vuint32m1_t ctr_le;
+ vuint32m1_t ctr_1;
+ vuint32m1_t ctr_2;
+ vuint32m1_t ctr_3;
+ vuint32m1_t ctr_4;
/* Byte swap counter */
- ctr_u64 = cast_u32m1_u64m1(bswap128_u32m1(ctr, vl));
+ ctr_le = bswap128_u32m1(ctr, vl);
/* Addition with carry handling */
- add_u64 = __riscv_vle64_v_u64m1(adders[idx][0], vl / 2);
- ctr_u64 = __riscv_vadd_vv_u64m1(ctr_u64, add_u64, vl / 2);
- ctr_u32_1 = cast_u64m1_u32m1(ctr_u64);
-
- add_u64 = __riscv_vle64_v_u64m1(adders[idx][1], vl / 2);
- ctr_u64 = __riscv_vadd_vv_u64m1(ctr_u64, add_u64, vl / 2);
- ctr_u32_2 = cast_u64m1_u32m1(ctr_u64);
-
- add_u64 = __riscv_vle64_v_u64m1(adders[idx][2], vl / 2);
- ctr_u64 = __riscv_vadd_vv_u64m1(ctr_u64, add_u64, vl / 2);
- ctr_u32_3 = cast_u64m1_u32m1(ctr_u64);
-
- add_u64 = __riscv_vle64_v_u64m1(adders[idx][3], vl / 2);
- ctr_u64 = __riscv_vadd_vv_u64m1(ctr_u64, add_u64, vl / 2);
- ctr_u32_4 = cast_u64m1_u32m1(ctr_u64);
+ ctr_1 = __riscv_vadd_vv_u32m1(ctr_le, add_1, vl);
+ ctr_2 = __riscv_vadd_vv_u32m1(ctr_le, add_2, vl);
+ ctr_3 = __riscv_vadd_vv_u32m1(ctr_1, add_2, vl);
+ ctr_4 = __riscv_vadd_vv_u32m1(ctr_2, add_2, vl);
/* Byte swap counters */
- ctr_u32_1 = bswap128_u32m1(ctr_u32_1, vl);
- ctr_u32_2 = bswap128_u32m1(ctr_u32_2, vl);
- ctr_u32_3 = bswap128_u32m1(ctr_u32_3, vl);
- ctr_u32_4 = bswap128_u32m1(ctr_u32_4, vl);
-
- ctr4blks = merge_4x_u32m1_to_u32m4(ctr, ctr_u32_1, ctr_u32_2,
- ctr_u32_3);
- ctr = ctr_u32_4;
+ ctr_1 = bswap128_u32m1(ctr_1, vl);
+ ctr_2 = bswap128_u32m1(ctr_2, vl);
+ ctr_3 = bswap128_u32m1(ctr_3, vl);
+ ctr_4 = bswap128_u32m1(ctr_4, vl);
+
+ ctr4blks = merge_4x_u32m1_to_u32m4(ctr, ctr_1, ctr_2, ctr_3);
+ ctr = ctr_4;
}
else
{
/* Fast path addition without carry handling */
- vuint8m1_t ctr_u8 = cast_u32m1_u8m1(ctr);
- vuint8m1_t ctr1 = __riscv_vadd_vv_u8m1(ctr_u8, add1, vl_bytes);
- vuint8m1_t ctr2 = __riscv_vadd_vv_u8m1(ctr_u8, add2, vl_bytes);
- vuint8m1_t ctr3 = __riscv_vadd_vv_u8m1(ctr_u8, add3, vl_bytes);
-
- ctr = cast_u8m1_u32m1(__riscv_vadd_vv_u8m1(ctr_u8, add4,
- vl_bytes));
-
- ctr4blks = merge_4x_u32m1_to_u32m4(cast_u8m1_u32m1(ctr_u8),
- cast_u8m1_u32m1(ctr1),
- cast_u8m1_u32m1(ctr2),
- cast_u8m1_u32m1(ctr3));
+ vuint32m1_t ctr0 = ctr;
+ vuint32m1_t ctr1 = __riscv_vadd_vv_u32m1(ctr, add1, vl);
+ vuint32m1_t ctr2 = __riscv_vadd_vv_u32m1(ctr, add2, vl);
+ vuint32m1_t ctr3 = __riscv_vadd_vv_u32m1(ctr, add3, vl);
+
+ ctr = __riscv_vadd_vv_u32m1(ctr, add4, vl);
+
+ ctr4blks = merge_4x_u32m1_to_u32m4(ctr0, ctr1, ctr2, ctr3);
}
data4blks = __riscv_vle8_v_u8m4(inbuf, vl_bytes * 4);
@@ -862,15 +840,13 @@ _gcry_aes_riscv_zvkned_ctr_enc (void *context, unsigned char *ctr_arg,
/* detect if 8-bit carry handling is needed */
if (UNLIKELY((++ctrlow & 0xff) == 0))
{
- const u64 *add_arr = UNLIKELY(ctrlow == 0) ? carry_add : nocarry_add;
- vuint64m1_t add_val = __riscv_vle64_v_u64m1(add_arr, vl / 2);
+ vuint32m1_t add_val = __riscv_vle32_v_u32m1(le_add, vl);
/* Byte swap counter */
ctr = bswap128_u32m1(ctr, vl);
/* Addition with carry handling */
- ctr = cast_u64m1_u32m1(__riscv_vadd_vv_u64m1(cast_u32m1_u64m1(ctr),
- add_val, vl / 2));
+ ctr = __riscv_vadd_vv_u32m1(ctr, add_val, vl);
/* Byte swap counter */
ctr = bswap128_u32m1(ctr, vl);
@@ -878,8 +854,7 @@ _gcry_aes_riscv_zvkned_ctr_enc (void *context, unsigned char *ctr_arg,
else
{
/* Fast path addition without carry handling */
- ctr = cast_u8m1_u32m1(__riscv_vadd_vv_u8m1(cast_u32m1_u8m1(ctr),
- add1, vl_bytes));
+ ctr = __riscv_vadd_vv_u32m1(ctr, add1, vl);
}
AES_CRYPT(e, m1, rounds, block, vl);
diff --git a/cipher/rijndael-ssse3-amd64.c b/cipher/rijndael-ssse3-amd64.c
index 0f0abf62..d39173a3 100644
--- a/cipher/rijndael-ssse3-amd64.c
+++ b/cipher/rijndael-ssse3-amd64.c
@@ -375,19 +375,10 @@ _gcry_aes_ssse3_ctr_enc (RIJNDAEL_context *ctx, unsigned char *ctr,
{
asm volatile ("movdqa %%xmm7, %%xmm0\n\t" /* xmm0 := CTR (xmm7) */
"pcmpeqd %%xmm1, %%xmm1\n\t"
- "psrldq $8, %%xmm1\n\t" /* xmm1 = -1 */
+ "psrldq $14, %%xmm1\n\t" /* xmm1 = -1 */
"pshufb %%xmm6, %%xmm7\n\t"
- "psubq %%xmm1, %%xmm7\n\t" /* xmm7++ (big endian) */
-
- /* detect if 64-bit carry handling is needed */
- "incq %q[ctrlow]\n\t"
- "jnz .Lno_carry%=\n\t"
-
- "pslldq $8, %%xmm1\n\t" /* move lower 64-bit to high */
- "psubq %%xmm1, %%xmm7\n\t" /* add carry to upper 64bits */
-
- ".Lno_carry%=:\n\t"
+ "psubw %%xmm1, %%xmm7\n\t" /* xmm7++ (big endian) */
"pshufb %%xmm6, %%xmm7\n\t"
: [ctrlow] "+r" (ctrlow)
diff --git a/cipher/rijndael-vaes-avx2-amd64.S b/cipher/rijndael-vaes-avx2-amd64.S
index 07e6f1ca..65fd79fe 100644
--- a/cipher/rijndael-vaes-avx2-amd64.S
+++ b/cipher/rijndael-vaes-avx2-amd64.S
@@ -714,40 +714,14 @@ _gcry_vaes_avx2_ctr_enc_amd64:
*/
CFI_STARTPROC();
- movq 8(%rsi), %r10;
- movq 0(%rsi), %r11;
- bswapq %r10;
- bswapq %r11;
+ movzbl 15(%rsi), %r10d;
+ movzbl 14(%rsi), %r11d;
vpcmpeqd %ymm15, %ymm15, %ymm15;
- vpsrldq $8, %ymm15, %ymm15; // 0:-1
- vpaddq %ymm15, %ymm15, %ymm14; // 0:-2
+ vpsrldq $14, %ymm15, %ymm15; // 0:-1
+ vpaddw %ymm15, %ymm15, %ymm14; // 0:-2
vbroadcasti128 .Lbswap128_mask rRIP, %ymm13;
-#define inc_le128(x, minus_one, tmp) \
- vpcmpeqq minus_one, x, tmp; \
- vpsubq minus_one, x, x; \
- vpslldq $8, tmp, tmp; \
- vpsubq tmp, x, x;
-
-#define add2_le128(x, minus_one, minus_two, tmp1, tmp2) \
- vpcmpeqq minus_one, x, tmp1; \
- vpcmpeqq minus_two, x, tmp2; \
- vpor tmp1, tmp2, tmp2; \
- vpsubq minus_two, x, x; \
- vpslldq $8, tmp2, tmp2; \
- vpsubq tmp2, x, x;
-
-#define handle_ctr_128bit_add(nblks) \
- addq $(nblks), %r10; \
- adcq $0, %r11; \
- bswapq %r10; \
- bswapq %r11; \
- movq %r10, 8(%rsi); \
- movq %r11, 0(%rsi); \
- bswapq %r10; \
- bswapq %r11;
-
/* Process 16 blocks per loop. */
.align 8
.Lctr_enc_blk16:
@@ -760,11 +734,9 @@ _gcry_vaes_avx2_ctr_enc_amd64:
vbroadcasti128 (0 * 16)(%rdi), %ymm8;
/* detect if carry handling is needed */
- addb $16, 15(%rsi);
+ addb $16, %r10b;
jc .Lctr_enc_blk16_handle_carry;
- leaq 16(%r10), %r10;
-
.Lctr_enc_blk16_byte_bige_add:
/* Increment counters. */
vpaddb .Lbige_addb_0 rRIP, %ymm7, %ymm0;
@@ -777,6 +749,7 @@ _gcry_vaes_avx2_ctr_enc_amd64:
vpaddb .Lbige_addb_14 rRIP, %ymm7, %ymm7;
.Lctr_enc_blk16_rounds:
+ movb %r10b, 15(%rsi);
/* AES rounds */
XOR8(%ymm8, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7);
vbroadcasti128 (1 * 16)(%rdi), %ymm8;
@@ -841,34 +814,30 @@ _gcry_vaes_avx2_ctr_enc_amd64:
jmp .Lctr_enc_blk16;
- .align 8
- .Lctr_enc_blk16_handle_only_ctr_carry:
- handle_ctr_128bit_add(16);
- jmp .Lctr_enc_blk16_byte_bige_add;
-
.align 8
.Lctr_enc_blk16_handle_carry:
- jz .Lctr_enc_blk16_handle_only_ctr_carry;
- /* Increment counters (handle carry). */
+ leal 1(%r11d), %r11d;
+ movb %r11b, 14(%rsi);
+ jz .Lctr_enc_blk16_byte_bige_add;
+
+ /* Increment counters (handle 16-bit carry). */
vpshufb %xmm13, %xmm7, %xmm1; /* be => le */
- vmovdqa %xmm1, %xmm0;
- inc_le128(%xmm1, %xmm15, %xmm5);
- vinserti128 $1, %xmm1, %ymm0, %ymm7; /* ctr: +1:+0 */
+ vpsubw %xmm15, %xmm1, %xmm0;
+ vinserti128 $1, %xmm0, %ymm1, %ymm7; /* ctr: +1:+0 */
vpshufb %ymm13, %ymm7, %ymm0;
- handle_ctr_128bit_add(16);
- add2_le128(%ymm7, %ymm15, %ymm14, %ymm9, %ymm10); /* ctr: +3:+2 */
+ vpsubw %ymm14, %ymm7, %ymm7; /* ctr: +3:+2 */
vpshufb %ymm13, %ymm7, %ymm1;
- add2_le128(%ymm7, %ymm15, %ymm14, %ymm9, %ymm10); /* ctr: +5:+4 */
+ vpsubw %ymm14, %ymm7, %ymm7; /* ctr: +5:+4 */
vpshufb %ymm13, %ymm7, %ymm2;
- add2_le128(%ymm7, %ymm15, %ymm14, %ymm9, %ymm10); /* ctr: +7:+6 */
+ vpsubw %ymm14, %ymm7, %ymm7; /* ctr: +7:+6 */
vpshufb %ymm13, %ymm7, %ymm3;
- add2_le128(%ymm7, %ymm15, %ymm14, %ymm9, %ymm10); /* ctr: +9:+8 */
+ vpsubw %ymm14, %ymm7, %ymm7; /* ctr: +9:+8 */
vpshufb %ymm13, %ymm7, %ymm4;
- add2_le128(%ymm7, %ymm15, %ymm14, %ymm9, %ymm10); /* ctr: +11:+10 */
+ vpsubw %ymm14, %ymm7, %ymm7; /* ctr: +11:+10 */
vpshufb %ymm13, %ymm7, %ymm5;
- add2_le128(%ymm7, %ymm15, %ymm14, %ymm9, %ymm10); /* ctr: +13:+12 */
+ vpsubw %ymm14, %ymm7, %ymm7; /* ctr: +13:+12 */
vpshufb %ymm13, %ymm7, %ymm6;
- add2_le128(%ymm7, %ymm15, %ymm14, %ymm9, %ymm10); /* ctr: +15:+14 */
+ vpsubw %ymm14, %ymm7, %ymm7; /* ctr: +15:+14 */
vpshufb %ymm13, %ymm7, %ymm7;
jmp .Lctr_enc_blk16_rounds;
@@ -885,11 +854,9 @@ _gcry_vaes_avx2_ctr_enc_amd64:
vbroadcasti128 (0 * 16)(%rdi), %ymm4;
/* detect if carry handling is needed */
- addb $8, 15(%rsi);
+ addb $8, %r10b;
jc .Lctr_enc_blk8_handle_carry;
- leaq 8(%r10), %r10;
-
.Lctr_enc_blk8_byte_bige_add:
/* Increment counters. */
vpaddb .Lbige_addb_0 rRIP, %ymm3, %ymm0;
@@ -898,6 +865,7 @@ _gcry_vaes_avx2_ctr_enc_amd64:
vpaddb .Lbige_addb_6 rRIP, %ymm3, %ymm3;
.Lctr_enc_blk8_rounds:
+ movb %r10b, 15(%rsi);
/* AES rounds */
XOR4(%ymm4, %ymm0, %ymm1, %ymm2, %ymm3);
vbroadcasti128 (1 * 16)(%rdi), %ymm4;
@@ -950,26 +918,22 @@ _gcry_vaes_avx2_ctr_enc_amd64:
jmp .Lctr_enc_blk4;
- .align 8
- .Lctr_enc_blk8_handle_only_ctr_carry:
- handle_ctr_128bit_add(8);
- jmp .Lctr_enc_blk8_byte_bige_add;
-
.align 8
.Lctr_enc_blk8_handle_carry:
- jz .Lctr_enc_blk8_handle_only_ctr_carry;
- /* Increment counters (handle carry). */
+ leal 1(%r11d), %r11d;
+ movb %r11b, 14(%rsi);
+ jz .Lctr_enc_blk8_byte_bige_add;
+
+ /* Increment counters (handle 16-bit carry). */
vpshufb %xmm13, %xmm3, %xmm1; /* be => le */
- vmovdqa %xmm1, %xmm0;
- inc_le128(%xmm1, %xmm15, %xmm5);
- vinserti128 $1, %xmm1, %ymm0, %ymm3; /* ctr: +1:+0 */
+ vpsubw %xmm15, %xmm1, %xmm0;
+ vinserti128 $1, %xmm0, %ymm1, %ymm3; /* ctr: +1:+0 */
vpshufb %ymm13, %ymm3, %ymm0;
- handle_ctr_128bit_add(8);
- add2_le128(%ymm3, %ymm15, %ymm14, %ymm5, %ymm6); /* ctr: +3:+2 */
+ vpsubw %ymm14, %ymm3, %ymm3; /* ctr: +3:+2 */
vpshufb %ymm13, %ymm3, %ymm1;
- add2_le128(%ymm3, %ymm15, %ymm14, %ymm5, %ymm6); /* ctr: +5:+4 */
+ vpsubw %ymm14, %ymm3, %ymm3; /* ctr: +5:+4 */
vpshufb %ymm13, %ymm3, %ymm2;
- add2_le128(%ymm3, %ymm15, %ymm14, %ymm5, %ymm6); /* ctr: +7:+6 */
+ vpsubw %ymm14, %ymm3, %ymm3; /* ctr: +7:+6 */
vpshufb %ymm13, %ymm3, %ymm3;
jmp .Lctr_enc_blk8_rounds;
@@ -986,17 +950,16 @@ _gcry_vaes_avx2_ctr_enc_amd64:
vbroadcasti128 (0 * 16)(%rdi), %ymm4;
/* detect if carry handling is needed */
- addb $4, 15(%rsi);
+ addb $4, %r10b;
jc .Lctr_enc_blk4_handle_carry;
- leaq 4(%r10), %r10;
-
.Lctr_enc_blk4_byte_bige_add:
/* Increment counters. */
vpaddb .Lbige_addb_0 rRIP, %ymm3, %ymm0;
vpaddb .Lbige_addb_2 rRIP, %ymm3, %ymm1;
.Lctr_enc_blk4_rounds:
+ movb %r10b, 15(%rsi);
/* AES rounds */
XOR2(%ymm4, %ymm0, %ymm1);
vbroadcasti128 (1 * 16)(%rdi), %ymm4;
@@ -1043,22 +1006,18 @@ _gcry_vaes_avx2_ctr_enc_amd64:
jmp .Lctr_enc_blk1;
- .align 8
- .Lctr_enc_blk4_handle_only_ctr_carry:
- handle_ctr_128bit_add(4);
- jmp .Lctr_enc_blk4_byte_bige_add;
-
.align 8
.Lctr_enc_blk4_handle_carry:
- jz .Lctr_enc_blk4_handle_only_ctr_carry;
- /* Increment counters (handle carry). */
+ leal 1(%r11d), %r11d;
+ movb %r11b, 14(%rsi);
+ jz .Lctr_enc_blk4_byte_bige_add;
+
+ /* Increment counters (handle 16-bit carry). */
vpshufb %xmm13, %xmm3, %xmm1; /* be => le */
- vmovdqa %xmm1, %xmm0;
- inc_le128(%xmm1, %xmm15, %xmm5);
- vinserti128 $1, %xmm1, %ymm0, %ymm3; /* ctr: +1:+0 */
+ vpsubw %xmm15, %xmm1, %xmm0;
+ vinserti128 $1, %xmm0, %ymm1, %ymm3; /* ctr: +1:+0 */
vpshufb %ymm13, %ymm3, %ymm0;
- handle_ctr_128bit_add(4);
- add2_le128(%ymm3, %ymm15, %ymm14, %ymm5, %ymm6); /* ctr: +3:+2 */
+ vpsubw %ymm14, %ymm3, %ymm3; /* ctr: +3:+2 */
vpshufb %ymm13, %ymm3, %ymm1;
jmp .Lctr_enc_blk4_rounds;
@@ -1073,13 +1032,16 @@ _gcry_vaes_avx2_ctr_enc_amd64:
/* Load and increament counter. */
vmovdqu (%rsi), %xmm0;
- handle_ctr_128bit_add(1);
+ addb $1, %r10b;
+ adcb $0, %r11b;
/* AES rounds. */
vpxor (0 * 16)(%rdi), %xmm0, %xmm0;
vaesenc (1 * 16)(%rdi), %xmm0, %xmm0;
vaesenc (2 * 16)(%rdi), %xmm0, %xmm0;
vaesenc (3 * 16)(%rdi), %xmm0, %xmm0;
+ movb %r10b, 15(%rsi);
+ movb %r11b, 14(%rsi);
vaesenc (4 * 16)(%rdi), %xmm0, %xmm0;
vaesenc (5 * 16)(%rdi), %xmm0, %xmm0;
vaesenc (6 * 16)(%rdi), %xmm0, %xmm0;
diff --git a/cipher/rijndael-vaes-avx2-i386.S b/cipher/rijndael-vaes-avx2-i386.S
index 245e8443..d1bf8bb5 100644
--- a/cipher/rijndael-vaes-avx2-i386.S
+++ b/cipher/rijndael-vaes-avx2-i386.S
@@ -556,15 +556,15 @@ SYM_NAME(_gcry_vaes_avx2_ctr_enc_i386):
movl %esp, %ebp;
CFI_DEF_CFA_REGISTER(%ebp);
- subl $(3 * 32 + 3 * 4), %esp;
+ subl $(1 * 32 + 3 * 4), %esp;
andl $-32, %esp;
- movl %edi, (3 * 32 + 0 * 4)(%esp);
- CFI_REG_ON_STACK(edi, 3 * 32 + 0 * 4);
- movl %esi, (3 * 32 + 1 * 4)(%esp);
- CFI_REG_ON_STACK(esi, 3 * 32 + 1 * 4);
- movl %ebx, (3 * 32 + 2 * 4)(%esp);
- CFI_REG_ON_STACK(ebx, 3 * 32 + 2 * 4);
+ movl %edi, (1 * 32 + 0 * 4)(%esp);
+ CFI_REG_ON_STACK(edi, 1 * 32 + 0 * 4);
+ movl %esi, (1 * 32 + 1 * 4)(%esp);
+ CFI_REG_ON_STACK(esi, 1 * 32 + 1 * 4);
+ movl %ebx, (1 * 32 + 2 * 4)(%esp);
+ CFI_REG_ON_STACK(ebx, 1 * 32 + 2 * 4);
movl %eax, %ebx;
movl 4+4(%ebp), %edi;
@@ -574,50 +574,8 @@ SYM_NAME(_gcry_vaes_avx2_ctr_enc_i386):
#define prepare_ctr_const(minus_one, minus_two) \
vpcmpeqd minus_one, minus_one, minus_one; \
- vpsrldq $8, minus_one, minus_one; /* 0:-1 */ \
- vpaddq minus_one, minus_one, minus_two; /* 0:-2 */
-
-#define inc_le128(x, minus_one, tmp) \
- vpcmpeqq minus_one, x, tmp; \
- vpsubq minus_one, x, x; \
- vpslldq $8, tmp, tmp; \
- vpsubq tmp, x, x;
-
-#define add2_le128(x, minus_one, minus_two, tmp1, tmp2) \
- vpcmpeqq minus_one, x, tmp1; \
- vpcmpeqq minus_two, x, tmp2; \
- vpor tmp1, tmp2, tmp2; \
- vpsubq minus_two, x, x; \
- vpslldq $8, tmp2, tmp2; \
- vpsubq tmp2, x, x;
-
-#define handle_ctr_128bit_add(nblks) \
- movl 12(%esi), %eax; \
- bswapl %eax; \
- addl $nblks, %eax; \
- bswapl %eax; \
- movl %eax, 12(%esi); \
- jnc 1f; \
- \
- movl 8(%esi), %eax; \
- bswapl %eax; \
- adcl $0, %eax; \
- bswapl %eax; \
- movl %eax, 8(%esi); \
- \
- movl 4(%esi), %eax; \
- bswapl %eax; \
- adcl $0, %eax; \
- bswapl %eax; \
- movl %eax, 4(%esi); \
- \
- movl 0(%esi), %eax; \
- bswapl %eax; \
- adcl $0, %eax; \
- bswapl %eax; \
- movl %eax, 0(%esi); \
- .align 8; \
- 1:;
+ vpsrldq $14, minus_one, minus_one; /* 0:-1 */ \
+ vpaddw minus_one, minus_one, minus_two; /* 0:-2 */
cmpl $12, 4+20(%ebp);
jae .Lctr_enc_blk12_loop;
@@ -631,10 +589,8 @@ SYM_NAME(_gcry_vaes_avx2_ctr_enc_i386):
vbroadcasti128 (%esi), %ymm6;
/* detect if carry handling is needed */
- movl 12(%esi), %eax;
- addl $(12 << 24), %eax;
+ addb $12, 15(%esi);
jc .Lctr_enc_blk12_handle_carry;
- movl %eax, 12(%esi);
.Lctr_enc_blk12_byte_bige_add:
/* Increment counters. */
@@ -707,39 +663,32 @@ SYM_NAME(_gcry_vaes_avx2_ctr_enc_i386):
jae .Lctr_enc_blk12_loop;
jmp .Lctr_enc_blk4;
- .align 8
- .Lctr_enc_blk12_handle_only_ctr_carry:
- handle_ctr_128bit_add(12);
- jmp .Lctr_enc_blk12_byte_bige_add;
-
.align 8
.Lctr_enc_blk12_handle_carry:
- jz .Lctr_enc_blk12_handle_only_ctr_carry;
- /* Increment counters (handle carry). */
- prepare_ctr_const(%ymm4, %ymm7);
- vmovdqa CADDR(.Lbswap128_mask, %ebx), %ymm2;
- vpshufb %xmm2, %xmm6, %xmm1; /* be => le */
- vmovdqa %xmm1, %xmm0;
- inc_le128(%xmm1, %xmm4, %xmm5);
- vinserti128 $1, %xmm1, %ymm0, %ymm6; /* ctr: +1:+0 */
- handle_ctr_128bit_add(12);
- vpshufb %ymm2, %ymm6, %ymm0;
+ movzbl 14(%esi), %eax;
+ leal 1(%eax), %eax;
+ movb %al, 14(%esi);
+ jz .Lctr_enc_blk12_byte_bige_add;
+
+ /* Increment counters (handle 16-bit carry). */
+ prepare_ctr_const(%ymm2, %ymm7);
+ vmovdqa CADDR(.Lbswap128_mask, %ebx), %ymm4;
+ vpshufb %xmm4, %xmm6, %xmm1; /* be => le */
+ vpsubw %xmm2, %xmm1, %xmm0;
+ vinserti128 $1, %xmm0, %ymm1, %ymm6; /* ctr: +1:+0 */
+ vpshufb %ymm4, %ymm6, %ymm0;
vmovdqa %ymm0, (0 * 32)(%esp);
- add2_le128(%ymm6, %ymm4, %ymm7, %ymm5, %ymm1); /* ctr: +3:+2 */
- vpshufb %ymm2, %ymm6, %ymm0;
- vmovdqa %ymm0, (1 * 32)(%esp);
- add2_le128(%ymm6, %ymm4, %ymm7, %ymm5, %ymm1); /* ctr: +5:+4 */
- vpshufb %ymm2, %ymm6, %ymm0;
- vmovdqa %ymm0, (2 * 32)(%esp);
- add2_le128(%ymm6, %ymm4, %ymm7, %ymm5, %ymm1); /* ctr: +7:+6 */
- vpshufb %ymm2, %ymm6, %ymm3;
- add2_le128(%ymm6, %ymm4, %ymm7, %ymm5, %ymm1); /* ctr: +9:+8 */
- vpshufb %ymm2, %ymm6, %ymm5;
- add2_le128(%ymm6, %ymm4, %ymm7, %ymm2, %ymm1); /* ctr: +11:+10 */
+ vpsubw %ymm7, %ymm6, %ymm6; /* ctr: +3:+2 */
+ vpshufb %ymm4, %ymm6, %ymm1;
+ vpsubw %ymm7, %ymm6, %ymm6; /* ctr: +5:+4 */
+ vpshufb %ymm4, %ymm6, %ymm2;
+ vpsubw %ymm7, %ymm6, %ymm6; /* ctr: +7:+6 */
+ vpshufb %ymm4, %ymm6, %ymm3;
+ vpsubw %ymm7, %ymm6, %ymm6; /* ctr: +9:+8 */
+ vpshufb %ymm4, %ymm6, %ymm5;
+ vpsubw %ymm7, %ymm6, %ymm6; /* ctr: +11:+10 */
vmovdqa (0 * 32)(%esp), %ymm0;
- vmovdqa (1 * 32)(%esp), %ymm1;
- vmovdqa (2 * 32)(%esp), %ymm2;
- vpshufb CADDR(.Lbswap128_mask, %ebx), %ymm6, %ymm6;
+ vpshufb %ymm4, %ymm6, %ymm6;
jmp .Lctr_enc_blk12_rounds;
@@ -754,10 +703,8 @@ SYM_NAME(_gcry_vaes_avx2_ctr_enc_i386):
vbroadcasti128 (%esi), %ymm3;
/* detect if carry handling is needed */
- movl 12(%esi), %eax;
- addl $(4 << 24), %eax;
+ addb $4, 15(%esi);
jc .Lctr_enc_blk4_handle_carry;
- movl %eax, 12(%esi);
.Lctr_enc_blk4_byte_bige_add:
/* Increment counters. */
@@ -812,24 +759,22 @@ SYM_NAME(_gcry_vaes_avx2_ctr_enc_i386):
jmp .Lctr_enc_blk1;
- .align 8
- .Lctr_enc_blk4_handle_only_ctr_carry:
- handle_ctr_128bit_add(4);
- jmp .Lctr_enc_blk4_byte_bige_add;
-
.align 8
.Lctr_enc_blk4_handle_carry:
- jz .Lctr_enc_blk4_handle_only_ctr_carry;
- /* Increment counters (handle carry). */
+ movzbl 14(%esi), %eax;
+ leal 1(%eax), %eax;
+ movb %al, 14(%esi);
+ jz .Lctr_enc_blk4_byte_bige_add;
+
+ /* Increment counters (handle 16-bit carry). */
prepare_ctr_const(%ymm4, %ymm7);
- vpshufb CADDR(.Lbswap128_mask, %ebx), %xmm3, %xmm1; /* be => le */
- vmovdqa %xmm1, %xmm0;
- inc_le128(%xmm1, %xmm4, %xmm5);
- vinserti128 $1, %xmm1, %ymm0, %ymm3; /* ctr: +1:+0 */
- vpshufb CADDR(.Lbswap128_mask, %ebx), %ymm3, %ymm0;
- handle_ctr_128bit_add(4);
- add2_le128(%ymm3, %ymm4, %ymm7, %ymm5, %ymm6); /* ctr: +3:+2 */
- vpshufb CADDR(.Lbswap128_mask, %ebx), %ymm3, %ymm1;
+ vmovdqa CADDR(.Lbswap128_mask, %ebx), %ymm2;
+ vpshufb %xmm2, %xmm3, %xmm1; /* be => le */
+ vpsubw %xmm4, %xmm1, %xmm0;
+ vinserti128 $1, %xmm0, %ymm1, %ymm3; /* ctr: +1:+0 */
+ vpshufb %ymm2, %ymm3, %ymm0;
+ vpsubw %ymm7, %ymm3, %ymm3; /* ctr: +3:+2 */
+ vpshufb %ymm2, %ymm3, %ymm1;
jmp .Lctr_enc_blk4_rounds;
@@ -841,15 +786,16 @@ SYM_NAME(_gcry_vaes_avx2_ctr_enc_i386):
subl $1, 4+20(%ebp);
- /* Load and increament counter. */
+ /* Load counter. */
vmovdqu (%esi), %xmm0;
- handle_ctr_128bit_add(1);
/* AES rounds. */
vpxor (0 * 16)(%edi), %xmm0, %xmm0;
vaesenc (1 * 16)(%edi), %xmm0, %xmm0;
vaesenc (2 * 16)(%edi), %xmm0, %xmm0;
vaesenc (3 * 16)(%edi), %xmm0, %xmm0;
+ addb $1, 15(%esi); /* Increament counter. */
+ adcb $0, 14(%esi);
vaesenc (4 * 16)(%edi), %xmm0, %xmm0;
vaesenc (5 * 16)(%edi), %xmm0, %xmm0;
vaesenc (6 * 16)(%edi), %xmm0, %xmm0;
@@ -880,18 +826,17 @@ SYM_NAME(_gcry_vaes_avx2_ctr_enc_i386):
.align 8
.Ldone_ctr_enc:
vpxor %ymm0, %ymm0, %ymm0;
- movl (3 * 32 + 0 * 4)(%esp), %edi;
+ movl (1 * 32 + 0 * 4)(%esp), %edi;
CFI_RESTORE(edi);
- movl (3 * 32 + 1 * 4)(%esp), %esi;
+ movl (1 * 32 + 1 * 4)(%esp), %esi;
CFI_RESTORE(esi);
- movl (3 * 32 + 2 * 4)(%esp), %ebx;
+ movl (1 * 32 + 2 * 4)(%esp), %ebx;
CFI_RESTORE(ebx);
vmovdqa %ymm0, (0 * 32)(%esp);
- vmovdqa %ymm0, (1 * 32)(%esp);
- vmovdqa %ymm0, (2 * 32)(%esp);
leave;
CFI_LEAVE();
vzeroall;
+ xorl %eax, %eax;
ret_spec_stop
CFI_ENDPROC();
ELF(.size SYM_NAME(_gcry_vaes_avx2_ctr_enc_i386),
diff --git a/cipher/rijndael-vaes-avx512-amd64.S b/cipher/rijndael-vaes-avx512-amd64.S
index f20998b0..96cbdd0a 100644
--- a/cipher/rijndael-vaes-avx512-amd64.S
+++ b/cipher/rijndael-vaes-avx512-amd64.S
@@ -519,16 +519,14 @@ _gcry_vaes_avx512_ctr_enc_amd64:
spec_stop_avx512;
- movq 8(%rsi), %r10;
- movq 0(%rsi), %r11;
- bswapq %r10;
- bswapq %r11;
-
vmovdqa32 .Lbige_addb_0 rRIP, %zmm20;
vmovdqa32 .Lbige_addb_4 rRIP, %zmm21;
vmovdqa32 .Lbige_addb_8 rRIP, %zmm22;
vmovdqa32 .Lbige_addb_12 rRIP, %zmm23;
+ movzbl 15(%rsi), %r10d;
+ movzbl 14(%rsi), %r11d;
+
/* Load first and last key. */
leal (, %r9d, 4), %eax;
vbroadcasti32x4 (%rdi), %zmm30;
@@ -542,22 +540,6 @@ _gcry_vaes_avx512_ctr_enc_amd64:
vmovdqa32 .Lbige_addb_24 rRIP, %zmm26;
vmovdqa32 .Lbige_addb_28 rRIP, %zmm27;
-#define add_le128(out, in, lo_counter, hi_counter1) \
- vpaddq lo_counter, in, out; \
- vpcmpuq $1, lo_counter, out, %k1; \
- kaddb %k1, %k1, %k1; \
- vpaddq hi_counter1, out, out{%k1};
-
-#define handle_ctr_128bit_add(nblks) \
- addq $(nblks), %r10; \
- adcq $0, %r11; \
- bswapq %r10; \
- bswapq %r11; \
- movq %r10, 8(%rsi); \
- movq %r11, 0(%rsi); \
- bswapq %r10; \
- bswapq %r11;
-
/* Process 32 blocks per loop. */
.align 16
.Lctr_enc_blk32:
@@ -567,11 +549,9 @@ _gcry_vaes_avx512_ctr_enc_amd64:
vbroadcasti32x4 (1 * 16)(%rdi), %zmm8;
/* detect if carry handling is needed */
- addb $32, 15(%rsi);
+ addb $32, %r10b;
jc .Lctr_enc_blk32_handle_carry;
- leaq 32(%r10), %r10;
-
.Lctr_enc_blk32_byte_bige_add:
/* Increment counters. */
vpaddb %zmm20, %zmm7, %zmm0;
@@ -584,6 +564,7 @@ _gcry_vaes_avx512_ctr_enc_amd64:
vpaddb %zmm27, %zmm7, %zmm7;
.Lctr_enc_blk32_rounds:
+ movb %r10b, 15(%rsi);
/* AES rounds */
XOR8(%zmm30, %zmm0, %zmm1, %zmm2, %zmm3, %zmm4, %zmm5, %zmm6, %zmm7);
VAESENC8(%zmm8, %zmm0, %zmm1, %zmm2, %zmm3, %zmm4, %zmm5, %zmm6, %zmm7);
@@ -656,35 +637,31 @@ _gcry_vaes_avx512_ctr_enc_amd64:
jmp .Lctr_enc_blk16;
- .align 16
- .Lctr_enc_blk32_handle_only_ctr_carry:
- handle_ctr_128bit_add(32);
- jmp .Lctr_enc_blk32_byte_bige_add;
-
.align 16
.Lctr_enc_blk32_handle_carry:
- jz .Lctr_enc_blk32_handle_only_ctr_carry;
- /* Increment counters (handle carry). */
+ leal 1(%r11d), %r11d;
+ movb %r11b, 14(%rsi);
+ jz .Lctr_enc_blk32_byte_bige_add;
+
+ /* Increment counters (handle 16-bit carry). */
vbroadcasti32x4 .Lbswap128_mask rRIP, %zmm15;
vpmovzxbq .Lcounter0_1_2_3_lo_bq rRIP, %zmm10;
- vpmovzxbq .Lcounter1_1_1_1_hi_bq rRIP, %zmm13;
vpshufb %zmm15, %zmm7, %zmm7; /* be => le */
vpmovzxbq .Lcounter4_4_4_4_lo_bq rRIP, %zmm11;
vpmovzxbq .Lcounter8_8_8_8_lo_bq rRIP, %zmm12;
- handle_ctr_128bit_add(32);
- add_le128(%zmm0, %zmm7, %zmm10, %zmm13); /* +0:+1:+2:+3 */
- add_le128(%zmm1, %zmm0, %zmm11, %zmm13); /* +4:+5:+6:+7 */
- add_le128(%zmm2, %zmm0, %zmm12, %zmm13); /* +8:... */
+ vpaddw %zmm10, %zmm7, %zmm0; /* +0:+1:+2:+3 */
+ vpaddw %zmm11, %zmm0, %zmm1; /* +4:+5:+6:+7 */
+ vpaddw %zmm12, %zmm0, %zmm2; /* +8:... */
vpshufb %zmm15, %zmm0, %zmm0; /* le => be */
- add_le128(%zmm3, %zmm1, %zmm12, %zmm13); /* +12:... */
+ vpaddw %zmm12, %zmm1, %zmm3; /* +12:... */
vpshufb %zmm15, %zmm1, %zmm1; /* le => be */
- add_le128(%zmm4, %zmm2, %zmm12, %zmm13); /* +16:... */
+ vpaddw %zmm12, %zmm2, %zmm4; /* +16:... */
vpshufb %zmm15, %zmm2, %zmm2; /* le => be */
- add_le128(%zmm5, %zmm3, %zmm12, %zmm13); /* +20:... */
+ vpaddw %zmm12, %zmm3, %zmm5; /* +20:... */
vpshufb %zmm15, %zmm3, %zmm3; /* le => be */
- add_le128(%zmm6, %zmm4, %zmm12, %zmm13); /* +24:... */
+ vpaddw %zmm12, %zmm4, %zmm6; /* +24:... */
vpshufb %zmm15, %zmm4, %zmm4; /* le => be */
- add_le128(%zmm7, %zmm5, %zmm12, %zmm13); /* +28:... */
+ vpaddw %zmm12, %zmm5, %zmm7; /* +28:... */
vpshufb %zmm15, %zmm5, %zmm5; /* le => be */
vpshufb %zmm15, %zmm6, %zmm6; /* le => be */
vpshufb %zmm15, %zmm7, %zmm7; /* le => be */
@@ -703,11 +680,9 @@ _gcry_vaes_avx512_ctr_enc_amd64:
vbroadcasti32x4 (1 * 16)(%rdi), %zmm4;
/* detect if carry handling is needed */
- addb $16, 15(%rsi);
+ addb $16, %r10b;
jc .Lctr_enc_blk16_handle_carry;
- leaq 16(%r10), %r10;
-
.Lctr_enc_blk16_byte_bige_add:
/* Increment counters. */
vpaddb %zmm20, %zmm3, %zmm0;
@@ -716,6 +691,7 @@ _gcry_vaes_avx512_ctr_enc_amd64:
vpaddb %zmm23, %zmm3, %zmm3;
.Lctr_enc_blk16_rounds:
+ movb %r10b, 15(%rsi);
/* AES rounds */
XOR4(%zmm30, %zmm0, %zmm1, %zmm2, %zmm3);
VAESENC4(%zmm4, %zmm0, %zmm1, %zmm2, %zmm3);
@@ -767,27 +743,23 @@ _gcry_vaes_avx512_ctr_enc_amd64:
jmp .Lctr_enc_tail;
- .align 16
- .Lctr_enc_blk16_handle_only_ctr_carry:
- handle_ctr_128bit_add(16);
- jmp .Lctr_enc_blk16_byte_bige_add;
-
.align 16
.Lctr_enc_blk16_handle_carry:
- jz .Lctr_enc_blk16_handle_only_ctr_carry;
- /* Increment counters (handle carry). */
+ leal 1(%r11d), %r11d;
+ movb %r11b, 14(%rsi);
+ jz .Lctr_enc_blk16_byte_bige_add;
+
+ /* Increment counters (handle 16-bit carry). */
vbroadcasti32x4 .Lbswap128_mask rRIP, %zmm15;
vpmovzxbq .Lcounter0_1_2_3_lo_bq rRIP, %zmm10;
- vpmovzxbq .Lcounter1_1_1_1_hi_bq rRIP, %zmm13;
vpshufb %zmm15, %zmm3, %zmm3; /* be => le */
vpmovzxbq .Lcounter4_4_4_4_lo_bq rRIP, %zmm11;
vpmovzxbq .Lcounter8_8_8_8_lo_bq rRIP, %zmm12;
- handle_ctr_128bit_add(16);
- add_le128(%zmm0, %zmm3, %zmm10, %zmm13); /* +0:+1:+2:+3 */
- add_le128(%zmm1, %zmm0, %zmm11, %zmm13); /* +4:+5:+6:+7 */
- add_le128(%zmm2, %zmm0, %zmm12, %zmm13); /* +8:... */
+ vpaddw %zmm10, %zmm3, %zmm0; /* +0:+1:+2:+3 */
+ vpaddw %zmm11, %zmm0, %zmm1; /* +4:+5:+6:+7 */
+ vpaddw %zmm12, %zmm0, %zmm2; /* +8:... */
vpshufb %zmm15, %zmm0, %zmm0; /* le => be */
- add_le128(%zmm3, %zmm1, %zmm12, %zmm13); /* +12:... */
+ vpaddw %zmm12, %zmm1, %zmm3; /* +12:... */
vpshufb %zmm15, %zmm1, %zmm1; /* le => be */
vpshufb %zmm15, %zmm2, %zmm2; /* le => be */
vpshufb %zmm15, %zmm3, %zmm3; /* le => be */
@@ -806,7 +778,6 @@ _gcry_vaes_avx512_ctr_enc_amd64:
vpxord %ymm23, %ymm23, %ymm23;
vpxord %ymm30, %ymm30, %ymm30;
vpxord %ymm31, %ymm31, %ymm31;
- kxorq %k1, %k1, %k1;
vzeroall;
.align 16
@@ -2462,8 +2433,6 @@ _gcry_vaes_avx512_consts:
.byte 16, 0, 16, 0, 16, 0, 16, 0
.Lcounter32_32_32_32_lo_bq:
.byte 32, 0, 32, 0, 32, 0, 32, 0
-.Lcounter1_1_1_1_hi_bq:
- .byte 0, 1, 0, 1, 0, 1, 0, 1
ELF(.size _gcry_vaes_avx512_consts,.-_gcry_vaes_avx512_consts)
diff --git a/cipher/rijndael-vp-simd128.h b/cipher/rijndael-vp-simd128.h
index 805faa6f..84dab316 100644
--- a/cipher/rijndael-vp-simd128.h
+++ b/cipher/rijndael-vp-simd128.h
@@ -1732,8 +1732,7 @@ FUNC_CTR_ENC (RIJNDAEL_context *ctx, unsigned char *ctr,
M128I_BYTE(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
static const __m128i_const bigendian_add =
M128I_BYTE(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1);
- static const __m128i_const carry_add = M128I_U64(1, 1);
- static const __m128i_const nocarry_add = M128I_U64(1, 0);
+ static const __m128i_const littleendian_add = M128I_U64(1, 0);
u64 ctrlow = buf_get_be64(ctr + 8);
struct vp_aes_config_s config;
@@ -1758,32 +1757,20 @@ FUNC_CTR_ENC (RIJNDAEL_context *ctx, unsigned char *ctr,
/* detect if 8-bit carry handling is needed */
if (UNLIKELY(((ctrlow += 4) & 0xff) <= 3))
{
- static const __m128i_const *adders[5][4] =
- {
- { &nocarry_add, &nocarry_add, &nocarry_add, &carry_add },
- { &nocarry_add, &nocarry_add, &carry_add, &nocarry_add },
- { &nocarry_add, &carry_add, &nocarry_add, &nocarry_add },
- { &carry_add, &nocarry_add, &nocarry_add, &nocarry_add },
- { &nocarry_add, &nocarry_add, &nocarry_add, &nocarry_add }
- };
- unsigned int idx = ctrlow <= 3 ? ctrlow : 4;
-
pshufb128(xmm6, xmm7);
-
- paddq128_amemld(adders[idx][0], xmm7);
+ paddd128_amemld(&littleendian_add, xmm7);
movdqa128(xmm7, xmm2);
pshufb128(xmm6, xmm2);
insert256_hi128(xmm2, ymm0);
- paddq128_amemld(adders[idx][1], xmm7);
+ paddd128_amemld(&littleendian_add, xmm7);
movdqa128(xmm7, xmm2);
pshufb128(xmm6, xmm2);
movdqa128_256(xmm2, ymm1);
- paddq128_amemld(adders[idx][2], xmm7);
+ paddd128_amemld(&littleendian_add, xmm7);
movdqa128(xmm7, xmm2);
pshufb128(xmm6, xmm2);
insert256_hi128(xmm2, ymm1);
- paddq128_amemld(adders[idx][3], xmm7);
-
+ paddd128_amemld(&littleendian_add, xmm7);
pshufb128(xmm6, xmm7);
}
else
@@ -1822,30 +1809,10 @@ FUNC_CTR_ENC (RIJNDAEL_context *ctx, unsigned char *ctr,
if (UNLIKELY(((ctrlow += 2) & 0xff) <= 1))
{
pshufb128(xmm6, xmm7);
-
- /* detect if 64-bit carry handling is needed */
- if (UNLIKELY(ctrlow == 1))
- {
- paddq128_amemld(&carry_add, xmm7);
- movdqa128(xmm7, xmm1);
- pshufb128(xmm6, xmm1);
- paddq128_amemld(&nocarry_add, xmm7);
- }
- else if (UNLIKELY(ctrlow == 0))
- {
- paddq128_amemld(&nocarry_add, xmm7);
- movdqa128(xmm7, xmm1);
- pshufb128(xmm6, xmm1);
- paddq128_amemld(&carry_add, xmm7);
- }
- else
- {
- paddq128_amemld(&nocarry_add, xmm7);
- movdqa128(xmm7, xmm1);
- pshufb128(xmm6, xmm1);
- paddq128_amemld(&nocarry_add, xmm7);
- }
-
+ paddd128_amemld(&littleendian_add, xmm7);
+ movdqa128(xmm7, xmm1);
+ paddq128_amemld(&littleendian_add, xmm7);
+ pshufb128(xmm6, xmm1);
pshufb128(xmm6, xmm7);
}
else
@@ -1877,10 +1844,7 @@ FUNC_CTR_ENC (RIJNDAEL_context *ctx, unsigned char *ctr,
if (UNLIKELY((++ctrlow & 0xff) == 0))
{
pshufb128(xmm6, xmm7);
-
- /* detect if 64-bit carry handling is needed */
- paddq128_amemld(UNLIKELY(ctrlow == 0) ? &carry_add : &nocarry_add, xmm7);
-
+ paddd128_amemld(&littleendian_add, xmm7);
pshufb128(xmm6, xmm7);
}
else
diff --git a/cipher/rijndael.c b/cipher/rijndael.c
index 645c0e2f..cf7f4c23 100644
--- a/cipher/rijndael.c
+++ b/cipher/rijndael.c
@@ -691,7 +691,7 @@ do_setkey (RIJNDAEL_context *ctx, const byte *key, const unsigned keylen,
bulk_ops->cfb_dec = _gcry_aes_cfb_dec;
bulk_ops->cbc_enc = _gcry_aes_cbc_enc;
bulk_ops->cbc_dec = _gcry_aes_cbc_dec;
- bulk_ops->ctr_enc = _gcry_aes_ctr_enc;
+ bulk_ops->ctr16be_enc = _gcry_aes_ctr_enc;
bulk_ops->ocb_crypt = _gcry_aes_ocb_crypt;
bulk_ops->ocb_auth = _gcry_aes_ocb_auth;
bulk_ops->xts_crypt = _gcry_aes_xts_crypt;
@@ -719,7 +719,7 @@ do_setkey (RIJNDAEL_context *ctx, const byte *key, const unsigned keylen,
bulk_ops->cfb_dec = _gcry_aes_aesni_cfb_dec;
bulk_ops->cbc_enc = _gcry_aes_aesni_cbc_enc;
bulk_ops->cbc_dec = _gcry_aes_aesni_cbc_dec;
- bulk_ops->ctr_enc = _gcry_aes_aesni_ctr_enc;
+ bulk_ops->ctr16be_enc = _gcry_aes_aesni_ctr_enc;
bulk_ops->ctr32le_enc = _gcry_aes_aesni_ctr32le_enc;
bulk_ops->ocb_crypt = _gcry_aes_aesni_ocb_crypt;
bulk_ops->ocb_auth = _gcry_aes_aesni_ocb_auth;
@@ -737,7 +737,7 @@ do_setkey (RIJNDAEL_context *ctx, const byte *key, const unsigned keylen,
/* Setup VAES bulk encryption routines. */
bulk_ops->cfb_dec = _gcry_aes_vaes_cfb_dec;
bulk_ops->cbc_dec = _gcry_aes_vaes_cbc_dec;
- bulk_ops->ctr_enc = _gcry_aes_vaes_ctr_enc;
+ bulk_ops->ctr16be_enc = _gcry_aes_vaes_ctr_enc;
bulk_ops->ctr32le_enc = _gcry_aes_vaes_ctr32le_enc;
bulk_ops->ocb_crypt = _gcry_aes_vaes_ocb_crypt;
bulk_ops->ocb_auth = _gcry_aes_vaes_ocb_auth;
@@ -752,7 +752,7 @@ do_setkey (RIJNDAEL_context *ctx, const byte *key, const unsigned keylen,
/* Setup VAES bulk encryption routines. */
bulk_ops->cfb_dec = _gcry_aes_vaes_cfb_dec;
bulk_ops->cbc_dec = _gcry_aes_vaes_cbc_dec;
- bulk_ops->ctr_enc = _gcry_aes_vaes_ctr_enc;
+ bulk_ops->ctr16be_enc = _gcry_aes_vaes_ctr_enc;
bulk_ops->ctr32le_enc = _gcry_aes_vaes_ctr32le_enc;
bulk_ops->ocb_crypt = _gcry_aes_vaes_ocb_crypt;
bulk_ops->ocb_auth = _gcry_aes_vaes_ocb_auth;
@@ -788,7 +788,7 @@ do_setkey (RIJNDAEL_context *ctx, const byte *key, const unsigned keylen,
bulk_ops->cfb_dec = _gcry_aes_ssse3_cfb_dec;
bulk_ops->cbc_enc = _gcry_aes_ssse3_cbc_enc;
bulk_ops->cbc_dec = _gcry_aes_ssse3_cbc_dec;
- bulk_ops->ctr_enc = _gcry_aes_ssse3_ctr_enc;
+ bulk_ops->ctr16be_enc = _gcry_aes_ssse3_ctr_enc;
bulk_ops->ocb_crypt = _gcry_aes_ssse3_ocb_crypt;
bulk_ops->ocb_auth = _gcry_aes_ssse3_ocb_auth;
}
@@ -808,7 +808,7 @@ do_setkey (RIJNDAEL_context *ctx, const byte *key, const unsigned keylen,
bulk_ops->cfb_dec = _gcry_aes_armv8_ce_cfb_dec;
bulk_ops->cbc_enc = _gcry_aes_armv8_ce_cbc_enc;
bulk_ops->cbc_dec = _gcry_aes_armv8_ce_cbc_dec;
- bulk_ops->ctr_enc = _gcry_aes_armv8_ce_ctr_enc;
+ bulk_ops->ctr16be_enc = _gcry_aes_armv8_ce_ctr_enc;
bulk_ops->ctr32le_enc = _gcry_aes_armv8_ce_ctr32le_enc;
bulk_ops->ocb_crypt = _gcry_aes_armv8_ce_ocb_crypt;
bulk_ops->ocb_auth = _gcry_aes_armv8_ce_ocb_auth;
@@ -831,7 +831,7 @@ do_setkey (RIJNDAEL_context *ctx, const byte *key, const unsigned keylen,
bulk_ops->cfb_dec = _gcry_aes_vp_aarch64_cfb_dec;
bulk_ops->cbc_enc = _gcry_aes_vp_aarch64_cbc_enc;
bulk_ops->cbc_dec = _gcry_aes_vp_aarch64_cbc_dec;
- bulk_ops->ctr_enc = _gcry_aes_vp_aarch64_ctr_enc;
+ bulk_ops->ctr16be_enc = _gcry_aes_vp_aarch64_ctr_enc;
bulk_ops->ctr32le_enc = _gcry_aes_vp_aarch64_ctr32le_enc;
bulk_ops->ocb_crypt = _gcry_aes_vp_aarch64_ocb_crypt;
bulk_ops->ocb_auth = _gcry_aes_vp_aarch64_ocb_auth;
@@ -858,7 +858,7 @@ do_setkey (RIJNDAEL_context *ctx, const byte *key, const unsigned keylen,
bulk_ops->cfb_dec = _gcry_aes_riscv_zvkned_cfb_dec;
bulk_ops->cbc_enc = _gcry_aes_riscv_zvkned_cbc_enc;
bulk_ops->cbc_dec = _gcry_aes_riscv_zvkned_cbc_dec;
- bulk_ops->ctr_enc = _gcry_aes_riscv_zvkned_ctr_enc;
+ bulk_ops->ctr16be_enc = _gcry_aes_riscv_zvkned_ctr_enc;
bulk_ops->ctr32le_enc = _gcry_aes_riscv_zvkned_ctr32le_enc;
bulk_ops->ocb_crypt = _gcry_aes_riscv_zvkned_ocb_crypt;
bulk_ops->ocb_auth = _gcry_aes_riscv_zvkned_ocb_auth;
@@ -884,7 +884,7 @@ do_setkey (RIJNDAEL_context *ctx, const byte *key, const unsigned keylen,
bulk_ops->cfb_dec = _gcry_aes_vp_riscv_cfb_dec;
bulk_ops->cbc_enc = _gcry_aes_vp_riscv_cbc_enc;
bulk_ops->cbc_dec = _gcry_aes_vp_riscv_cbc_dec;
- bulk_ops->ctr_enc = _gcry_aes_vp_riscv_ctr_enc;
+ bulk_ops->ctr16be_enc = _gcry_aes_vp_riscv_ctr_enc;
bulk_ops->ctr32le_enc = _gcry_aes_vp_riscv_ctr32le_enc;
bulk_ops->ocb_crypt = _gcry_aes_vp_riscv_ocb_crypt;
bulk_ops->ocb_auth = _gcry_aes_vp_riscv_ocb_auth;
@@ -908,7 +908,7 @@ do_setkey (RIJNDAEL_context *ctx, const byte *key, const unsigned keylen,
bulk_ops->cfb_dec = _gcry_aes_ppc9le_cfb_dec;
bulk_ops->cbc_enc = _gcry_aes_ppc9le_cbc_enc;
bulk_ops->cbc_dec = _gcry_aes_ppc9le_cbc_dec;
- bulk_ops->ctr_enc = _gcry_aes_ppc9le_ctr_enc;
+ bulk_ops->ctr16be_enc = _gcry_aes_ppc9le_ctr_enc;
bulk_ops->ocb_crypt = _gcry_aes_ppc9le_ocb_crypt;
bulk_ops->ocb_auth = _gcry_aes_ppc9le_ocb_auth;
bulk_ops->xts_crypt = _gcry_aes_ppc9le_xts_crypt;
@@ -939,7 +939,7 @@ do_setkey (RIJNDAEL_context *ctx, const byte *key, const unsigned keylen,
bulk_ops->cfb_dec = _gcry_aes_ppc8_cfb_dec;
bulk_ops->cbc_enc = _gcry_aes_ppc8_cbc_enc;
bulk_ops->cbc_dec = _gcry_aes_ppc8_cbc_dec;
- bulk_ops->ctr_enc = _gcry_aes_ppc8_ctr_enc;
+ bulk_ops->ctr16be_enc = _gcry_aes_ppc8_ctr_enc;
bulk_ops->ocb_crypt = _gcry_aes_ppc8_ocb_crypt;
bulk_ops->ocb_auth = _gcry_aes_ppc8_ocb_auth;
bulk_ops->xts_crypt = _gcry_aes_ppc8_xts_crypt;
@@ -1348,7 +1348,7 @@ _gcry_aes_ctr_enc (void *context, unsigned char *ctr,
outbuf += BLOCKSIZE;
inbuf += BLOCKSIZE;
/* Increment the counter. */
- cipher_block_add(ctr, 1, BLOCKSIZE);
+ cipher_block_add_be16(ctr, 1, BLOCKSIZE);
}
wipememory(&tmp, sizeof(tmp));
diff --git a/tests/basic.c b/tests/basic.c
index 1f4273ea..9ec1b596 100644
--- a/tests/basic.c
+++ b/tests/basic.c
@@ -12666,6 +12666,10 @@ cipher_cbc_bulk_test (int cipher_algo)
return -1;
}
+ if (verbose)
+ fprintf (stderr, " checking CBC bulk encryption for %s [%i]\n",
+ cipher, cipher_algo);
+
memsize = (blocksize * 2) + (blocksize * nblocks * 3) + 16 + (blocksize + 1);
mem = xcalloc (1, memsize);
@@ -12906,6 +12910,10 @@ cipher_cfb_bulk_test (int cipher_algo)
return -1;
}
+ if (verbose)
+ fprintf (stderr, " checking CFB bulk encryption for %s [%i]\n",
+ cipher, cipher_algo);
+
memsize = (blocksize * 2) + (blocksize * nblocks * 3) + 16 + (blocksize + 1);
mem = xcalloc (1, memsize);
@@ -13135,6 +13143,10 @@ cipher_ctr_bulk_test (int cipher_algo)
return -1;
}
+ if (verbose)
+ fprintf (stderr, " checking CTR bulk encryption for %s [%i]\n",
+ cipher, cipher_algo);
+
memsize = (blocksize * 2) + (blocksize * nblocks * 4) + 16 + (blocksize + 1);
mem = xcalloc (1, memsize);
@@ -13440,6 +13452,132 @@ cipher_ctr_bulk_test (int cipher_algo)
}
+/* Regression test for the 16-bit-overflow split in bulk CTR encryption: a
+ single call spanning >= 0x10000 blocks must not reuse keystream nor drop the
+ counter carry. Reference keystream is built from ECB with a full-width
+ byte-wise counter, correct independent of the split logic. */
+static int
+cipher_ctr16_overflow_test (int cipher_algo)
+{
+ static const unsigned int low16_start[] =
+ { 0, 1, 0x7fff, 0x8000, 0xfffe, 0xffff };
+ const size_t nblocks = 2 * 0x10000 + 5;
+ int blocksize;
+ const char *cipher;
+ gcry_cipher_hd_t hd_ecb = NULL;
+ gcry_cipher_hd_t hd_ctr = NULL;
+ unsigned char *plaintext = NULL;
+ unsigned char *reftext = NULL;
+ unsigned char *enctext = NULL;
+ unsigned char iv[16];
+ unsigned char getctr[17];
+ size_t buflen, i, j, tc;
+ unsigned int keylen;
+ static const unsigned char key[32] = {
+ 0x06,0x9A,0x00,0x7F,0xC7,0x6A,0x45,0x9F,
+ 0x98,0xBA,0xF9,0x17,0xFE,0xDF,0x95,0x21,
+ 0x06,0x9A,0x00,0x7F,0xC7,0x6A,0x45,0x9F,
+ 0x98,0xBA,0xF9,0x17,0xFE,0xDF,0x95,0x21
+ };
+
+ if (gcry_cipher_test_algo (cipher_algo))
+ return 0;
+ blocksize = gcry_cipher_get_algo_blklen (cipher_algo);
+ if (blocksize < 8 || blocksize > 16)
+ return 0;
+ cipher = gcry_cipher_algo_name (cipher_algo);
+ keylen = gcry_cipher_get_algo_keylen (cipher_algo);
+ if (keylen > sizeof(key))
+ return 0;
+
+ if (verbose)
+ fprintf (stderr, " checking CTR 16-bit overflow for %s [%i]\n",
+ cipher, cipher_algo);
+
+ buflen = nblocks * (size_t)blocksize;
+ plaintext = xmalloc (buflen);
+ reftext = xmalloc (buflen);
+ enctext = xmalloc (buflen);
+
+ if (gcry_cipher_open (&hd_ecb, cipher_algo, GCRY_CIPHER_MODE_ECB, 0)
+ || gcry_cipher_open (&hd_ctr, cipher_algo, GCRY_CIPHER_MODE_CTR, 0)
+ || gcry_cipher_setkey (hd_ecb, key, keylen)
+ || gcry_cipher_setkey (hd_ctr, key, keylen))
+ {
+ fail ("%s-CTR-%d ctr16 test failed (setup)", cipher, blocksize * 8);
+ goto leave;
+ }
+
+ for (i = 0; i < buflen; i++)
+ plaintext[i] = (unsigned char)i;
+
+ for (tc = 0; tc < DIM (low16_start); tc++)
+ {
+ unsigned int low16 = low16_start[tc];
+
+ /* Fixed non-zero upper bytes exercise carry past the low 16 bits. */
+ memset (iv, 0x5a, blocksize);
+ iv[blocksize - 3] = 0x40;
+ iv[blocksize - 2] = (low16 >> 8) & 0xff;
+ iv[blocksize - 1] = low16 & 0xff;
+
+ if (gcry_cipher_setctr (hd_ctr, iv, blocksize))
+ {
+ fail ("%s-CTR-%d ctr16 test failed (setctr, low16=0x%04x)",
+ cipher, blocksize * 8, low16);
+ goto leave;
+ }
+
+ /* iv ends holding start + nblocks, the reference final counter. */
+ for (i = 0; i < buflen; i += blocksize)
+ {
+ memcpy (&reftext[i], iv, blocksize);
+ for (j = blocksize; j > 0; j--)
+ if (++iv[j - 1])
+ break;
+ }
+ if (gcry_cipher_encrypt (hd_ecb, reftext, buflen, NULL, 0))
+ {
+ fail ("%s-CTR-%d ctr16 test failed (ecb, low16=0x%04x)",
+ cipher, blocksize * 8, low16);
+ goto leave;
+ }
+ for (i = 0; i < buflen; i++)
+ reftext[i] ^= plaintext[i];
+
+ if (gcry_cipher_encrypt (hd_ctr, enctext, buflen, plaintext, buflen))
+ {
+ fail ("%s-CTR-%d ctr16 test failed (ctr, low16=0x%04x)",
+ cipher, blocksize * 8, low16);
+ goto leave;
+ }
+ if (memcmp (enctext, reftext, buflen))
+ {
+ fail ("%s-CTR-%d ctr16 test failed (keystream, low16=0x%04x)",
+ cipher, blocksize * 8, low16);
+ goto leave;
+ }
+
+ if (gcry_cipher_ctl (hd_ctr, PRIV_CIPHERCTL_GET_COUNTER, getctr,
+ blocksize + 1)
+ || getctr[0] != blocksize
+ || memcmp (getctr + 1, iv, blocksize))
+ {
+ fail ("%s-CTR-%d ctr16 test failed (counter, low16=0x%04x)",
+ cipher, blocksize * 8, low16);
+ goto leave;
+ }
+ }
+
+leave:
+ gcry_cipher_close (hd_ecb);
+ gcry_cipher_close (hd_ctr);
+ xfree (plaintext);
+ xfree (reftext);
+ xfree (enctext);
+ return 0;
+}
+
static void
check_ciphers (void)
@@ -13603,6 +13741,20 @@ check_ciphers (void)
}
+static void
+check_ctr16_overflow (void)
+{
+ /* One 128-bit and one 64-bit block cipher cover both blocksize code paths
+ in _gcry_cipher_ctr_encrypt_ctx. */
+ if (verbose)
+ fprintf (stderr, " Starting CTR 16-bit overflow checks.\n");
+ cipher_ctr16_overflow_test (GCRY_CIPHER_AES);
+ cipher_ctr16_overflow_test (GCRY_CIPHER_BLOWFISH);
+ if (verbose)
+ fprintf (stderr, " Completed CTR 16-bit overflow checks.\n");
+}
+
+
static void
check_cipher_modes(void)
{
@@ -13613,6 +13765,8 @@ check_cipher_modes(void)
check_aes128_cbc_cts_cipher ();
check_cbc_mac_cipher ();
check_ctr_cipher ();
+ check_ctr16_overflow ();
+
check_cfb_cipher ();
check_ofb_cipher ();
check_ccm_cipher ();
--
2.53.0
More information about the Gcrypt-devel
mailing list