[PATCH 2/2] cipher: limit table prefetch to the look-up table arrays

[Jussi Kivilinna]

* cipher/aria.c (prefetch_sboxes): Compute unshare counter once
and prefetch only look-up arrays.
* cipher/cipher-gcm.c (do_prefetch_tables): Likewise.
* cipher/rijndael.c (prefetch_enc, prefetch_dec): Likewise.
* cipher/sm4.c (prefetch_sbox_table): Likewise.
* cipher/rijndael-tables.h (dec_tables): Rename 'inv_sbox' field to
'inv_sboxT'.
--

Signed-off-by: Jussi Kivilinna <jussi.kivilinna at iki.fi>
---
 cipher/aria.c            |  7 ++++---
 cipher/cipher-gcm.c      |  7 ++++---
 cipher/rijndael-tables.h |  4 ++--
 cipher/rijndael.c        | 15 +++++++++------
 cipher/sm4.c             |  7 ++++---
 5 files changed, 23 insertions(+), 17 deletions(-)

diff --git a/cipher/aria.c b/cipher/aria.c
index 26546a63..bb67ed03 100644
--- a/cipher/aria.c
+++ b/cipher/aria.c
@@ -627,11 +627,12 @@ prefetch_sboxes(void)
    * of look-up table are shared between processes.  Modifying counters also
    * causes checksums for pages to change and hint same-page merging algorithm
    * that these pages are frequently changing.  */
-  sboxes.counter_head++;
-  sboxes.counter_tail++;
+  u32 counter = sboxes.counter_head + 1;
+  sboxes.counter_head = counter;
+  sboxes.counter_tail = counter;
 
   /* Prefetch look-up tables to cache.  */
-  prefetch_table((const void *)&sboxes, sizeof(sboxes));
+  prefetch_table((const void *)&sboxes.s1[0], sizeof(sboxes.s1) * 4);
 }
 
 
diff --git a/cipher/cipher-gcm.c b/cipher/cipher-gcm.c
index 2db371b8..1627cd1c 100644
--- a/cipher/cipher-gcm.c
+++ b/cipher/cipher-gcm.c
@@ -255,12 +255,13 @@ do_prefetch_tables (const void *gcmM, size_t gcmM_size)
    * of look-up table are shared between processes.  Modifying counters also
    * causes checksums for pages to change and hint same-page merging algorithm
    * that these pages are frequently changing.  */
-  gcm_table.counter_head++;
-  gcm_table.counter_tail++;
+  u32 counter = gcm_table.counter_head + 1;
+  gcm_table.counter_head = counter;
+  gcm_table.counter_tail = counter;
 
   /* Prefetch look-up tables to cache.  */
   prefetch_table(gcmM, gcmM_size);
-  prefetch_table(&gcm_table, sizeof(gcm_table));
+  prefetch_table(&gcm_table.R, sizeof(gcm_table.R));
 }
 
 #ifdef GCM_TABLES_USE_U64
diff --git a/cipher/rijndael-tables.h b/cipher/rijndael-tables.h
index e46ce08c..52b9518f 100644
--- a/cipher/rijndael-tables.h
+++ b/cipher/rijndael-tables.h
@@ -107,7 +107,7 @@ static struct
   volatile u32 counter_head;
   u32 cacheline_align[64 / 4 - 1];
   u32 T[256];
-  byte inv_sbox[256];
+  byte inv_sboxT[256];
   volatile u32 counter_tail;
 } dec_tables ATTR_ALIGNED_64 =
   {
@@ -217,4 +217,4 @@ static struct
   };
 
 #define decT dec_tables.T
-#define inv_sbox dec_tables.inv_sbox
+#define inv_sbox dec_tables.inv_sboxT
diff --git a/cipher/rijndael.c b/cipher/rijndael.c
index f3daf35a..645c0e2f 100644
--- a/cipher/rijndael.c
+++ b/cipher/rijndael.c
@@ -597,11 +597,12 @@ static void prefetch_enc(void)
    * of look-up table are shared between processes.  Modifying counters also
    * causes checksums for pages to change and hint same-page merging algorithm
    * that these pages are frequently changing.  */
-  enc_tables.counter_head++;
-  enc_tables.counter_tail++;
+  u32 counter = enc_tables.counter_head + 1;
+  enc_tables.counter_head = counter;
+  enc_tables.counter_tail = counter;
 
   /* Prefetch look-up tables to cache.  */
-  prefetch_table((const void *)&enc_tables, sizeof(enc_tables));
+  prefetch_table((const void *)&enc_tables.T[0], sizeof(enc_tables.T));
 }
 
 static void prefetch_dec(void)
@@ -610,11 +611,13 @@ static void prefetch_dec(void)
    * of look-up table are shared between processes.  Modifying counters also
    * causes checksums for pages to change and hint same-page merging algorithm
    * that these pages are frequently changing.  */
-  dec_tables.counter_head++;
-  dec_tables.counter_tail++;
+  u32 counter = dec_tables.counter_head + 1;
+  dec_tables.counter_head = counter;
+  dec_tables.counter_tail = counter;
 
   /* Prefetch look-up tables to cache.  */
-  prefetch_table((const void *)&dec_tables, sizeof(dec_tables));
+  prefetch_table((const void *)&dec_tables.T[0],
+		 sizeof(dec_tables.T) + sizeof(dec_tables.inv_sboxT));
 }
 
 
diff --git a/cipher/sm4.c b/cipher/sm4.c
index cc98b676..70b4901c 100644
--- a/cipher/sm4.c
+++ b/cipher/sm4.c
@@ -760,14 +760,15 @@ sm4_ppc9le_crypt_blk1_16(void *rk, byte *out, const byte *in, size_t num_blks)
 
 static inline void prefetch_sbox_table(void)
 {
-  const volatile byte *vtab = (void *)&sbox_table;
+  const volatile byte *vtab = (void *)&sbox_table.S[0];
 
   /* Modify counters to trigger copy-on-write and unsharing if physical pages
    * of look-up table are shared between processes.  Modifying counters also
    * causes checksums for pages to change and hint same-page merging algorithm
    * that these pages are frequently changing.  */
-  sbox_table.counter_head++;
-  sbox_table.counter_tail++;
+  u32 counter = sbox_table.counter_head + 1;
+  sbox_table.counter_head = counter;
+  sbox_table.counter_tail = counter;
 
   /* Prefetch look-up table to cache.  */
   (void)vtab[0 * 32];
-- 
2.53.0