[PATCH 08/10] twofish: reduce CTR bulk counter carry handling to 16 bits
Jussi Kivilinna
jussi.kivilinna at iki.fi
Fri Jul 24 20:50:15 CEST 2026
* cipher/twofish.c (twofish_setkey): Assign ctr16be_enc bulk op.
(_gcry_twofish_ctr_enc): Use cipher_block_add_be16.
* cipher/twofish-amd64.S (RT0bl, RT1bl): New.
(_gcry_twofish_amd64_ctr_enc): Increment low 16 counter bits in place,
drop full-width carry and byteswap.
* cipher/twofish-avx2-amd64.S (inc_le128): Remove.
(_gcry_twofish_avx2_ctr_enc): Add to low 16 counter bits only, drop
full-width carry handling.
--
Twofish CTR built IVs with full 128-bit increment and byteswap
round-trip. Under ctr16be_enc contract generic ctr code splits work at
low 16-bit overflow, so bulk function adds only to low 16 counter bits.
Scalar amd64 path now increments low counter bytes in place, dropping
byteswap. AVX2 path uses 16-bit stepping and drops full-width carry.
Moves twofish off transitional ctr_enc alias.
Signed-off-by: Jussi Kivilinna <jussi.kivilinna at iki.fi>
---
cipher/twofish-amd64.S | 53 +++++++++++++--------------
cipher/twofish-avx2-amd64.S | 71 +++++++------------------------------
cipher/twofish.c | 4 +--
3 files changed, 39 insertions(+), 89 deletions(-)
diff --git a/cipher/twofish-amd64.S b/cipher/twofish-amd64.S
index 913b252d..beafd10f 100644
--- a/cipher/twofish-amd64.S
+++ b/cipher/twofish-amd64.S
@@ -359,6 +359,9 @@ ELF(.size _gcry_twofish_amd64_encrypt_block,.-_gcry_twofish_amd64_encrypt_block;
#define RT0d %edx
#define RT1d %esi
+#define RT0bl %dl
+#define RT1bl %sil
+
#define do16bit_ror(rot, op1, op2, T0, T1, tmp1, tmp2, ab, dst) \
movzbl ab ## bl, tmp2 ## d; \
movzbl ab ## bh, tmp1 ## d; \
@@ -650,35 +653,27 @@ _gcry_twofish_amd64_ctr_enc:
movq %rdx, (7 * 8)(%rsp);
movq %rcx, RX0;
- /* load IV and byteswap */
- movq 8(RX0), RT0;
- movq 0(RX0), RT1;
- movq RT0, RCD0;
- movq RT1, RAB0;
- bswapq RT0;
- bswapq RT1;
-
- /* construct IVs */
- movq RT0, RCD1;
- movq RT1, RAB1;
- movq RT0, RCD2;
- movq RT1, RAB2;
- addq $1, RCD1;
- adcq $0, RAB1;
- bswapq RCD1;
- bswapq RAB1;
- addq $2, RCD2;
- adcq $0, RAB2;
- bswapq RCD2;
- bswapq RAB2;
- addq $3, RT0;
- adcq $0, RT1;
- bswapq RT0;
- bswapq RT1;
-
- /* store new IV */
- movq RT0, 8(RX0);
- movq RT1, 0(RX0);
+ /* Construct and update IV */
+ movzbl 15(RX0), RT0d;
+ movzbl 14(RX0), RT1d;
+ movq 8(RX0), RCD0;
+ movq 0(RX0), RAB0;
+ movq RAB0, RAB1;
+ movq RAB0, RAB2;
+ addb $1, RT0bl;
+ adcb $0, RT1bl;
+ movb RT0bl, 15(RX0);
+ movb RT1bl, 14(RX0);
+ movq 8(RX0), RCD1;
+ addb $1, RT0bl;
+ adcb $0, RT1bl;
+ movb RT0bl, 15(RX0);
+ movb RT1bl, 14(RX0);
+ movq 8(RX0), RCD2;
+ addb $1, RT0bl;
+ adcb $0, RT1bl;
+ movb RT0bl, 15(RX0);
+ movb RT1bl, 14(RX0);
call __twofish_enc_blk3;
diff --git a/cipher/twofish-avx2-amd64.S b/cipher/twofish-avx2-amd64.S
index 67c1e52e..72a39d17 100644
--- a/cipher/twofish-avx2-amd64.S
+++ b/cipher/twofish-avx2-amd64.S
@@ -553,12 +553,6 @@ _gcry_twofish_avx2_blk16:
CFI_ENDPROC();
ELF(.size _gcry_twofish_avx2_blk16,.-_gcry_twofish_avx2_blk16;)
-#define inc_le128(x, minus_one, tmp) \
- vpcmpeqq minus_one, x, tmp; \
- vpsubq minus_one, x, x; \
- vpslldq $8, tmp, tmp; \
- vpsubq tmp, x, x;
-
.align 16
.globl _gcry_twofish_avx2_ctr_enc
ELF(.type _gcry_twofish_avx2_ctr_enc, at function;)
@@ -571,79 +565,40 @@ _gcry_twofish_avx2_ctr_enc:
*/
CFI_STARTPROC();
- movq 8(%rcx), %rax;
- bswapq %rax;
-
vzeroupper;
vbroadcasti128 .Lbswap128_mask rRIP, RTMP3;
vpcmpeqd RNOT, RNOT, RNOT;
- vpsrldq $8, RNOT, RNOT; /* ab: -1:0 ; cd: -1:0 */
- vpaddq RNOT, RNOT, RTMP2; /* ab: -2:0 ; cd: -2:0 */
+ vpsrldq $14, RNOT, RNOT; /* ab: -1:0 ; cd: -1:0 */
+ vpaddw RNOT, RNOT, RTMP2; /* ab: -2:0 ; cd: -2:0 */
/* load IV and byteswap */
vmovdqu (%rcx), RTMP4x;
vpshufb RTMP3x, RTMP4x, RTMP4x;
vmovdqa RTMP4x, RTMP0x;
- inc_le128(RTMP4x, RNOTx, RTMP1x);
+ vpsubw RNOTx, RTMP4x, RTMP4x;
vinserti128 $1, RTMP4x, RTMP0, RTMP0;
vpshufb RTMP3, RTMP0, RA0; /* +1 ; +0 */
- /* check need for handling 64-bit overflow and carry */
- cmpq $(0xffffffffffffffff - 16), %rax;
- ja .Lhandle_ctr_carry;
+ /* Update IV */
+ addb $16, 15(%rcx);
+ adcb $0, 14(%rcx);
/* construct IVs */
- vpsubq RTMP2, RTMP0, RTMP0; /* +3 ; +2 */
+ vpsubw RTMP2, RTMP0, RTMP0; /* +3 ; +2 */
vpshufb RTMP3, RTMP0, RB0;
- vpsubq RTMP2, RTMP0, RTMP0; /* +5 ; +4 */
+ vpsubw RTMP2, RTMP0, RTMP0; /* +5 ; +4 */
vpshufb RTMP3, RTMP0, RC0;
- vpsubq RTMP2, RTMP0, RTMP0; /* +7 ; +6 */
+ vpsubw RTMP2, RTMP0, RTMP0; /* +7 ; +6 */
vpshufb RTMP3, RTMP0, RD0;
- vpsubq RTMP2, RTMP0, RTMP0; /* +9 ; +8 */
+ vpsubw RTMP2, RTMP0, RTMP0; /* +9 ; +8 */
vpshufb RTMP3, RTMP0, RA1;
- vpsubq RTMP2, RTMP0, RTMP0; /* +11 ; +10 */
+ vpsubw RTMP2, RTMP0, RTMP0; /* +11 ; +10 */
vpshufb RTMP3, RTMP0, RB1;
- vpsubq RTMP2, RTMP0, RTMP0; /* +13 ; +12 */
+ vpsubw RTMP2, RTMP0, RTMP0; /* +13 ; +12 */
vpshufb RTMP3, RTMP0, RC1;
- vpsubq RTMP2, RTMP0, RTMP0; /* +15 ; +14 */
+ vpsubw RTMP2, RTMP0, RTMP0; /* +15 ; +14 */
vpshufb RTMP3, RTMP0, RD1;
- vpsubq RTMP2, RTMP0, RTMP0; /* +16 */
- vpshufb RTMP3x, RTMP0x, RTMP0x;
-
- jmp .Lctr_carry_done;
-
-.Lhandle_ctr_carry:
- /* construct IVs */
- inc_le128(RTMP0, RNOT, RTMP1);
- inc_le128(RTMP0, RNOT, RTMP1);
- vpshufb RTMP3, RTMP0, RB0; /* +3 ; +2 */
- inc_le128(RTMP0, RNOT, RTMP1);
- inc_le128(RTMP0, RNOT, RTMP1);
- vpshufb RTMP3, RTMP0, RC0; /* +5 ; +4 */
- inc_le128(RTMP0, RNOT, RTMP1);
- inc_le128(RTMP0, RNOT, RTMP1);
- vpshufb RTMP3, RTMP0, RD0; /* +7 ; +6 */
- inc_le128(RTMP0, RNOT, RTMP1);
- inc_le128(RTMP0, RNOT, RTMP1);
- vpshufb RTMP3, RTMP0, RA1; /* +9 ; +8 */
- inc_le128(RTMP0, RNOT, RTMP1);
- inc_le128(RTMP0, RNOT, RTMP1);
- vpshufb RTMP3, RTMP0, RB1; /* +11 ; +10 */
- inc_le128(RTMP0, RNOT, RTMP1);
- inc_le128(RTMP0, RNOT, RTMP1);
- vpshufb RTMP3, RTMP0, RC1; /* +13 ; +12 */
- inc_le128(RTMP0, RNOT, RTMP1);
- inc_le128(RTMP0, RNOT, RTMP1);
- vpshufb RTMP3, RTMP0, RD1; /* +15 ; +14 */
- inc_le128(RTMP0, RNOT, RTMP1);
- vextracti128 $1, RTMP0, RTMP0x;
- vpshufb RTMP3x, RTMP0x, RTMP0x; /* +16 */
-
-.align 4
-.Lctr_carry_done:
- /* store new IV */
- vmovdqu RTMP0x, (%rcx);
call __twofish_enc_blk16;
diff --git a/cipher/twofish.c b/cipher/twofish.c
index 11a6e251..4341a7fb 100644
--- a/cipher/twofish.c
+++ b/cipher/twofish.c
@@ -774,7 +774,7 @@ twofish_setkey (void *context, const byte *key, unsigned int keylen,
memset (bulk_ops, 0, sizeof(*bulk_ops));
bulk_ops->cbc_dec = _gcry_twofish_cbc_dec;
bulk_ops->cfb_dec = _gcry_twofish_cfb_dec;
- bulk_ops->ctr_enc = _gcry_twofish_ctr_enc;
+ bulk_ops->ctr16be_enc = _gcry_twofish_ctr_enc;
bulk_ops->ocb_crypt = _gcry_twofish_ocb_crypt;
bulk_ops->ocb_auth = _gcry_twofish_ocb_auth;
bulk_ops->xts_crypt = _gcry_twofish_xts_crypt;
@@ -1197,7 +1197,7 @@ _gcry_twofish_ctr_enc(void *context, unsigned char *ctr, void *outbuf_arg,
outbuf += TWOFISH_BLOCKSIZE;
inbuf += TWOFISH_BLOCKSIZE;
/* Increment the counter. */
- cipher_block_add(ctr, 1, TWOFISH_BLOCKSIZE);
+ cipher_block_add_be16(ctr, 1, TWOFISH_BLOCKSIZE);
}
wipememory(tmpbuf, sizeof(tmpbuf));
--
2.53.0
More information about the Gcrypt-devel
mailing list