[PATCH 1/7] Add straight-line speculation hardening for function ends

[Jussi Kivilinna]

* cipher/asm-common-amd64.h (SPEC_STOP): New.
(CFI_ENDPROC): Emit straight-line speculation barrier at function end.
(ret_spec_stop): Use 'SPEC_STOP'.
* cipher/asm-common-i386.h (SPEC_STOP, CFI_ENDPROC, ret_spec_stop):
Likewise.
* cipher/asm-common-aarch64.h (SPEC_STOP, CFI_ENDPROC, ret_spec_stop):
Likewise.
* cipher/cast5-amd64.S (__cast5_dec_blk4): Move 'CFI_ENDPROC' after
'ret_spec_stop'.
* mpi/i386/syntax.h (SPEC_STOP): New.
(ret_spec_stop): Use 'SPEC_STOP'.
--

The 'ret_spec_stop' macro places a speculation barrier after 'ret'
instructions only. Emit the barrier from 'CFI_ENDPROC' as well so that
the end of every function is covered, and move the barrier instruction
into a new 'SPEC_STOP' macro shared by both.

Signed-off-by: Jussi Kivilinna <jussi.kivilinna at iki.fi>
---
 cipher/asm-common-aarch64.h | 9 ++++++---
 cipher/asm-common-amd64.h   | 9 ++++++---
 cipher/asm-common-i386.h    | 9 ++++++---
 cipher/cast5-amd64.S        | 2 +-
 mpi/i386/syntax.h           | 5 ++++-
 5 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/cipher/asm-common-aarch64.h b/cipher/asm-common-aarch64.h
index dde7366c..ec9b9b92 100644
--- a/cipher/asm-common-aarch64.h
+++ b/cipher/asm-common-aarch64.h
@@ -61,10 +61,13 @@
 # define AARCH64_PAC_PROPERTY_FLAG 0 /* No PAC */
 #endif
 
+/* straight-line speculation mitigation */
+#define SPEC_STOP dsb sy; isb
+
 #ifdef HAVE_GCC_ASM_CFI_DIRECTIVES
 /* CFI directives to emit DWARF stack unwinding information. */
 # define CFI_STARTPROC()            .cfi_startproc; AARCH64_HINT_BTI_C
-# define CFI_ENDPROC()              .cfi_endproc
+# define CFI_ENDPROC()              SPEC_STOP; .cfi_endproc
 # define CFI_REMEMBER_STATE()       .cfi_remember_state
 # define CFI_RESTORE_STATE()        .cfi_restore_state
 # define CFI_ADJUST_CFA_OFFSET(off) .cfi_adjust_cfa_offset off
@@ -106,7 +109,7 @@
 
 #else
 # define CFI_STARTPROC() AARCH64_HINT_BTI_C
-# define CFI_ENDPROC()
+# define CFI_ENDPROC() SPEC_STOP
 # define CFI_REMEMBER_STATE()
 # define CFI_RESTORE_STATE()
 # define CFI_ADJUST_CFA_OFFSET(off)
@@ -121,7 +124,7 @@
 
 /* 'ret' instruction replacement for straight-line speculation mitigation */
 #define ret_spec_stop \
-	ret; dsb sy; isb;
+	ret; SPEC_STOP;
 
 #define CLEAR_REG(reg) movi reg.16b, #0;
 
diff --git a/cipher/asm-common-amd64.h b/cipher/asm-common-amd64.h
index d0cc6426..a827c025 100644
--- a/cipher/asm-common-amd64.h
+++ b/cipher/asm-common-amd64.h
@@ -88,10 +88,13 @@
 #define ENDBRANCH /*_*/
 #endif
 
+/* straight-line speculation mitigation */
+#define SPEC_STOP int3
+
 #ifdef HAVE_GCC_ASM_CFI_DIRECTIVES
 /* CFI directives to emit DWARF stack unwinding information. */
 # define CFI_STARTPROC()            .cfi_startproc; ENDBRANCH
-# define CFI_ENDPROC()              .cfi_endproc
+# define CFI_ENDPROC()              SPEC_STOP; .cfi_endproc
 # define CFI_REMEMBER_STATE()       .cfi_remember_state
 # define CFI_RESTORE_STATE()        .cfi_restore_state
 # define CFI_ADJUST_CFA_OFFSET(off) .cfi_adjust_cfa_offset off
@@ -159,7 +162,7 @@
 
 #else
 # define CFI_STARTPROC() ENDBRANCH
-# define CFI_ENDPROC()
+# define CFI_ENDPROC() SPEC_STOP
 # define CFI_REMEMBER_STATE()
 # define CFI_RESTORE_STATE()
 # define CFI_ADJUST_CFA_OFFSET(off)
@@ -210,7 +213,7 @@
 
 /* 'ret' instruction replacement for straight-line speculation mitigation. */
 #define ret_spec_stop \
-	ret; int3;
+	ret; SPEC_STOP;
 
 /* This prevents speculative execution on old AVX512 CPUs, to prevent
  * speculative execution to AVX512 code. The vpopcntb instruction is
diff --git a/cipher/asm-common-i386.h b/cipher/asm-common-i386.h
index 346a8ff2..9e85a54e 100644
--- a/cipher/asm-common-i386.h
+++ b/cipher/asm-common-i386.h
@@ -65,10 +65,13 @@
 #define ENDBRANCH /*_*/
 #endif
 
+/* straight-line speculation mitigation */
+#define SPEC_STOP int3
+
 #ifdef HAVE_GCC_ASM_CFI_DIRECTIVES
 /* CFI directives to emit DWARF stack unwinding information. */
 # define CFI_STARTPROC()            .cfi_startproc; ENDBRANCH
-# define CFI_ENDPROC()              .cfi_endproc
+# define CFI_ENDPROC()              SPEC_STOP; .cfi_endproc
 # define CFI_REMEMBER_STATE()       .cfi_remember_state
 # define CFI_RESTORE_STATE()        .cfi_restore_state
 # define CFI_ADJUST_CFA_OFFSET(off) .cfi_adjust_cfa_offset off
@@ -128,7 +131,7 @@
 
 #else
 # define CFI_STARTPROC() ENDBRANCH
-# define CFI_ENDPROC()
+# define CFI_ENDPROC() SPEC_STOP
 # define CFI_REMEMBER_STATE()
 # define CFI_RESTORE_STATE()
 # define CFI_ADJUST_CFA_OFFSET(off)
@@ -148,7 +151,7 @@
 
 /* 'ret' instruction replacement for straight-line speculation mitigation. */
 #define ret_spec_stop \
-	ret; int3;
+	ret; SPEC_STOP;
 
 /* This prevents speculative execution on old AVX512 CPUs, to prevent
  * speculative execution to AVX512 code. The vpopcntb instruction is
diff --git a/cipher/cast5-amd64.S b/cipher/cast5-amd64.S
index b8ae8ba0..aba88c16 100644
--- a/cipher/cast5-amd64.S
+++ b/cipher/cast5-amd64.S
@@ -431,8 +431,8 @@ __cast5_dec_blk4:
 	round_dec_last4(1, F4_2, F4_1);
 
 	outbswap_block4(RLR0, RLR1, RLR2, RLR3);
-	CFI_ENDPROC();
 	ret_spec_stop;
+	CFI_ENDPROC();
 ELF(.size __cast5_dec_blk4,.-__cast5_dec_blk4;)
 
 .align 16
diff --git a/mpi/i386/syntax.h b/mpi/i386/syntax.h
index 2383ebd3..acecb8d9 100644
--- a/mpi/i386/syntax.h
+++ b/mpi/i386/syntax.h
@@ -73,6 +73,9 @@
 #define ALIGN(log) .align log,0x90
 #endif
 
+/* straight-line speculation mitigation */
+#define SPEC_STOP int3
+
 /* 'ret' instruction replacement for straight-line speculation mitigation */
 #define ret_spec_stop \
-	ret; int3;
+	ret; SPEC_STOP;
-- 
2.53.0