commit 2cc16b98e2ae0ea34b94bfb4560973c303ede29f
parent adc0406051a470836d2e0bc2edb4b1fc2ca06c43
Author: Jared Tobin <jared@jtobin.io>
Date: Sat, 31 Oct 2026 13:53:46 +0400
cbits: add an x86 SHA512 extension kernel
On x86-64, sha512_block_native now uses the SHA512 extension
(vsha512rnds2, vsha512msg1, vsha512msg2) along with AVX2.
sha512_native_available checks both via CPUID, and that the OS saves
the YMM registers, so CPUs lacking them (e.g. Intel cores before
Arrow Lake and Lunar Lake) fall back to the pure Haskell
implementation. The K table is now shared between the
ARM and x86 kernels, and the stubs only serve other architectures.
The intrinsics first appeared in GCC 14 and Clang 18, so the x86
kernel is only compiled when the compiler provides <sha512intrin.h>;
older compilers get the stubs.
The instructions keep the state as (a,b,e,f) and (c,d,g,h), so the
kernel rearranges it on entry and exit; message words arrive in
native order and need no swapping.
Diffstat:
5 files changed, 260 insertions(+), 5 deletions(-)
diff --git a/README.md b/README.md
@@ -6,7 +6,7 @@
A pure Haskell implementation of SHA-512 and HMAC-SHA512 on strict and
lazy ByteStrings, as specified by RFC's [6234][r6234] and [2104][r2104],
-that uses ARM SHA2 intrinsics when available.
+that uses ARM SHA2 or x86 SHA512 intrinsics when available.
## Usage
diff --git a/cbits/sha512_native.c b/cbits/sha512_native.c
@@ -1,6 +1,16 @@
#include <stdint.h>
#include <string.h>
+/*
+ * The x86 SHA512 intrinsics arrived in GCC 14 and Clang 18; older
+ * compilers get the stubs below.
+ */
+#if defined(__x86_64__) && defined(__has_include)
+#if __has_include(<sha512intrin.h>)
+#define HAVE_X86_SHA512
+#endif
+#endif
+
#if defined(__aarch64__)
#include <arm_neon.h>
@@ -14,6 +24,19 @@
#endif
#endif
+#elif defined(HAVE_X86_SHA512)
+
+#include <cpuid.h>
+#include <immintrin.h>
+
+#ifndef bit_SHA512
+#define bit_SHA512 (1 << 0)
+#endif
+
+#endif
+
+#if defined(__aarch64__) || defined(HAVE_X86_SHA512)
+
static const uint64_t K[80] = {
0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL,
0xb5c0fbcfec4d3b2fULL, 0xe9b5dba58189dbbcULL,
@@ -57,6 +80,10 @@ static const uint64_t K[80] = {
0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL
};
+#endif
+
+#if defined(__aarch64__)
+
/*
* Process one 128-byte block using ARM SHA512 crypto instructions.
*
@@ -484,9 +511,236 @@ int sha512_native_available(void) {
#endif
}
+#elif defined(HAVE_X86_SHA512)
+
+/*
+ * Four rounds. 'sha512rnds2' does two: it takes the (c,d,g,h) and
+ * (a,b,e,f) halves of the state, and two message-plus-constant words
+ * in its third argument, and returns the new (a,b,e,f). The old
+ * (a,b,e,f) is then the new (c,d,g,h), so a second call with the
+ * halves swapped, on the upper two words, does the next two.
+ */
+#define ROUNDS4(abef, cdgh, msg) \
+ do { \
+ cdgh = _mm256_sha512rnds2_epi64(cdgh, abef, \
+ _mm256_castsi256_si128(msg)); \
+ abef = _mm256_sha512rnds2_epi64(abef, cdgh, \
+ _mm256_extracti128_si256(msg, 1)); \
+ } while (0)
+
+/*
+ * The four words following prev[0], i.e. (prev[1], prev[2], prev[3],
+ * cur[0]) from lowest lane to highest.
+ */
+#define ALIGNR1(cur, prev) \
+ _mm256_permute4x64_epi64(_mm256_blend_epi32(prev, cur, 0x03), 0x39)
+
+/*
+ * Process one 128-byte block using x86 SHA512 instructions.
+ *
+ * state: pointer to 8 uint64_t words (a,b,c,d,e,f,g,h)
+ * block: pointer to 16 uint64_t words (already native endian)
+ *
+ * The state is updated in place.
+ *
+ * Compiled for the SHA512 and AVX2 extensions only; callers must first
+ * check 'sha512_native_available'.
+ *
+ * Lane orders in comments are written from highest lane to lowest.
+ */
+__attribute__((target("avx2,sha512")))
+void sha512_block_native(uint64_t *state, const uint64_t *block) {
+ __m256i abcd, efgh, abef, cdgh, abef_orig, cdgh_orig;
+ __m256i m0, m1, m2, m3, msg;
+
+ /* Load current hash state, rearranging it for the SHA instructions */
+ abcd = _mm256_loadu_si256((const __m256i *)&state[0]); /* d c b a */
+ efgh = _mm256_loadu_si256((const __m256i *)&state[4]); /* h g f e */
+ abef = _mm256_permute4x64_epi64(
+ _mm256_permute2x128_si256(abcd, efgh, 0x20), 0x1b);
+ /* a b e f */
+ cdgh = _mm256_permute4x64_epi64(
+ _mm256_permute2x128_si256(abcd, efgh, 0x31), 0x1b);
+ /* c d g h */
+
+ /* Save original for final addition */
+ abef_orig = abef;
+ cdgh_orig = cdgh;
+
+ /* Load message (already native endian) */
+ m0 = _mm256_loadu_si256((const __m256i *)&block[0]);
+ m1 = _mm256_loadu_si256((const __m256i *)&block[4]);
+ m2 = _mm256_loadu_si256((const __m256i *)&block[8]);
+ m3 = _mm256_loadu_si256((const __m256i *)&block[12]);
+
+ /* Rounds 0-3 */
+ msg = _mm256_add_epi64(m0, _mm256_loadu_si256((const __m256i *)&K[0]));
+ ROUNDS4(abef, cdgh, msg);
+
+ /* Rounds 4-7 */
+ msg = _mm256_add_epi64(m1, _mm256_loadu_si256((const __m256i *)&K[4]));
+ ROUNDS4(abef, cdgh, msg);
+ m0 = _mm256_sha512msg1_epi64(m0, _mm256_castsi256_si128(m1));
+
+ /* Rounds 8-11 */
+ msg = _mm256_add_epi64(m2, _mm256_loadu_si256((const __m256i *)&K[8]));
+ ROUNDS4(abef, cdgh, msg);
+ m1 = _mm256_sha512msg1_epi64(m1, _mm256_castsi256_si128(m2));
+
+ /* Rounds 12-15 */
+ msg = _mm256_add_epi64(m3, _mm256_loadu_si256((const __m256i *)&K[12]));
+ ROUNDS4(abef, cdgh, msg);
+ m0 = _mm256_add_epi64(m0, ALIGNR1(m3, m2));
+ m0 = _mm256_sha512msg2_epi64(m0, m3);
+ m2 = _mm256_sha512msg1_epi64(m2, _mm256_castsi256_si128(m3));
+
+ /* Rounds 16-19 */
+ msg = _mm256_add_epi64(m0, _mm256_loadu_si256((const __m256i *)&K[16]));
+ ROUNDS4(abef, cdgh, msg);
+ m1 = _mm256_add_epi64(m1, ALIGNR1(m0, m3));
+ m1 = _mm256_sha512msg2_epi64(m1, m0);
+ m3 = _mm256_sha512msg1_epi64(m3, _mm256_castsi256_si128(m0));
+
+ /* Rounds 20-23 */
+ msg = _mm256_add_epi64(m1, _mm256_loadu_si256((const __m256i *)&K[20]));
+ ROUNDS4(abef, cdgh, msg);
+ m2 = _mm256_add_epi64(m2, ALIGNR1(m1, m0));
+ m2 = _mm256_sha512msg2_epi64(m2, m1);
+ m0 = _mm256_sha512msg1_epi64(m0, _mm256_castsi256_si128(m1));
+
+ /* Rounds 24-27 */
+ msg = _mm256_add_epi64(m2, _mm256_loadu_si256((const __m256i *)&K[24]));
+ ROUNDS4(abef, cdgh, msg);
+ m3 = _mm256_add_epi64(m3, ALIGNR1(m2, m1));
+ m3 = _mm256_sha512msg2_epi64(m3, m2);
+ m1 = _mm256_sha512msg1_epi64(m1, _mm256_castsi256_si128(m2));
+
+ /* Rounds 28-31 */
+ msg = _mm256_add_epi64(m3, _mm256_loadu_si256((const __m256i *)&K[28]));
+ ROUNDS4(abef, cdgh, msg);
+ m0 = _mm256_add_epi64(m0, ALIGNR1(m3, m2));
+ m0 = _mm256_sha512msg2_epi64(m0, m3);
+ m2 = _mm256_sha512msg1_epi64(m2, _mm256_castsi256_si128(m3));
+
+ /* Rounds 32-35 */
+ msg = _mm256_add_epi64(m0, _mm256_loadu_si256((const __m256i *)&K[32]));
+ ROUNDS4(abef, cdgh, msg);
+ m1 = _mm256_add_epi64(m1, ALIGNR1(m0, m3));
+ m1 = _mm256_sha512msg2_epi64(m1, m0);
+ m3 = _mm256_sha512msg1_epi64(m3, _mm256_castsi256_si128(m0));
+
+ /* Rounds 36-39 */
+ msg = _mm256_add_epi64(m1, _mm256_loadu_si256((const __m256i *)&K[36]));
+ ROUNDS4(abef, cdgh, msg);
+ m2 = _mm256_add_epi64(m2, ALIGNR1(m1, m0));
+ m2 = _mm256_sha512msg2_epi64(m2, m1);
+ m0 = _mm256_sha512msg1_epi64(m0, _mm256_castsi256_si128(m1));
+
+ /* Rounds 40-43 */
+ msg = _mm256_add_epi64(m2, _mm256_loadu_si256((const __m256i *)&K[40]));
+ ROUNDS4(abef, cdgh, msg);
+ m3 = _mm256_add_epi64(m3, ALIGNR1(m2, m1));
+ m3 = _mm256_sha512msg2_epi64(m3, m2);
+ m1 = _mm256_sha512msg1_epi64(m1, _mm256_castsi256_si128(m2));
+
+ /* Rounds 44-47 */
+ msg = _mm256_add_epi64(m3, _mm256_loadu_si256((const __m256i *)&K[44]));
+ ROUNDS4(abef, cdgh, msg);
+ m0 = _mm256_add_epi64(m0, ALIGNR1(m3, m2));
+ m0 = _mm256_sha512msg2_epi64(m0, m3);
+ m2 = _mm256_sha512msg1_epi64(m2, _mm256_castsi256_si128(m3));
+
+ /* Rounds 48-51 */
+ msg = _mm256_add_epi64(m0, _mm256_loadu_si256((const __m256i *)&K[48]));
+ ROUNDS4(abef, cdgh, msg);
+ m1 = _mm256_add_epi64(m1, ALIGNR1(m0, m3));
+ m1 = _mm256_sha512msg2_epi64(m1, m0);
+ m3 = _mm256_sha512msg1_epi64(m3, _mm256_castsi256_si128(m0));
+
+ /* Rounds 52-55 */
+ msg = _mm256_add_epi64(m1, _mm256_loadu_si256((const __m256i *)&K[52]));
+ ROUNDS4(abef, cdgh, msg);
+ m2 = _mm256_add_epi64(m2, ALIGNR1(m1, m0));
+ m2 = _mm256_sha512msg2_epi64(m2, m1);
+ m0 = _mm256_sha512msg1_epi64(m0, _mm256_castsi256_si128(m1));
+
+ /* Rounds 56-59 */
+ msg = _mm256_add_epi64(m2, _mm256_loadu_si256((const __m256i *)&K[56]));
+ ROUNDS4(abef, cdgh, msg);
+ m3 = _mm256_add_epi64(m3, ALIGNR1(m2, m1));
+ m3 = _mm256_sha512msg2_epi64(m3, m2);
+ m1 = _mm256_sha512msg1_epi64(m1, _mm256_castsi256_si128(m2));
+
+ /* Rounds 60-63 */
+ msg = _mm256_add_epi64(m3, _mm256_loadu_si256((const __m256i *)&K[60]));
+ ROUNDS4(abef, cdgh, msg);
+ m0 = _mm256_add_epi64(m0, ALIGNR1(m3, m2));
+ m0 = _mm256_sha512msg2_epi64(m0, m3);
+ m2 = _mm256_sha512msg1_epi64(m2, _mm256_castsi256_si128(m3));
+
+ /* Rounds 64-67 */
+ msg = _mm256_add_epi64(m0, _mm256_loadu_si256((const __m256i *)&K[64]));
+ ROUNDS4(abef, cdgh, msg);
+ m1 = _mm256_add_epi64(m1, ALIGNR1(m0, m3));
+ m1 = _mm256_sha512msg2_epi64(m1, m0);
+ m3 = _mm256_sha512msg1_epi64(m3, _mm256_castsi256_si128(m0));
+
+ /* Rounds 68-71 */
+ msg = _mm256_add_epi64(m1, _mm256_loadu_si256((const __m256i *)&K[68]));
+ ROUNDS4(abef, cdgh, msg);
+ m2 = _mm256_add_epi64(m2, ALIGNR1(m1, m0));
+ m2 = _mm256_sha512msg2_epi64(m2, m1);
+
+ /* Rounds 72-75 */
+ msg = _mm256_add_epi64(m2, _mm256_loadu_si256((const __m256i *)&K[72]));
+ ROUNDS4(abef, cdgh, msg);
+ m3 = _mm256_add_epi64(m3, ALIGNR1(m2, m1));
+ m3 = _mm256_sha512msg2_epi64(m3, m2);
+
+ /* Rounds 76-79 */
+ msg = _mm256_add_epi64(m3, _mm256_loadu_si256((const __m256i *)&K[76]));
+ ROUNDS4(abef, cdgh, msg);
+
+ /* Add original state back */
+ abef = _mm256_add_epi64(abef, abef_orig);
+ cdgh = _mm256_add_epi64(cdgh, cdgh_orig);
+
+ /* Restore the state's original order and store result */
+ abef = _mm256_permute4x64_epi64(abef, 0x1b); /* f e b a */
+ cdgh = _mm256_permute4x64_epi64(cdgh, 0x1b); /* h g d c */
+ _mm256_storeu_si256((__m256i *)&state[0],
+ _mm256_permute2x128_si256(abef, cdgh, 0x20)); /* d c b a */
+ _mm256_storeu_si256((__m256i *)&state[4],
+ _mm256_permute2x128_si256(abef, cdgh, 0x31)); /* h g f e */
+}
+
+/*
+ * Return 1 if the CPU implements the x86 SHA512 extension and the AVX2
+ * instructions used alongside it, and the OS saves the YMM registers,
+ * 0 otherwise.
+ */
+int sha512_native_available(void) {
+ unsigned int eax, ebx, ecx, edx, xcr0;
+
+ if (!__get_cpuid(1, &eax, &ebx, &ecx, &edx))
+ return 0;
+ if (!(ecx & bit_OSXSAVE) || !(ecx & bit_AVX))
+ return 0;
+ /* XCR0 bits 1 and 2: the OS saves the XMM and YMM registers */
+ __asm__ ("xgetbv" : "=a"(xcr0), "=d"(edx) : "c"(0));
+ if ((xcr0 & 0x6) != 0x6)
+ return 0;
+ if (!__get_cpuid_count(7, 0, &eax, &ebx, &ecx, &edx))
+ return 0;
+ if (eax < 1 || !(ebx & bit_AVX2))
+ return 0;
+ __get_cpuid_count(7, 1, &eax, &ebx, &ecx, &edx);
+ return (eax & bit_SHA512) != 0;
+}
+
#else
-/* Stub implementations for non-aarch64 builds */
+/* Stub implementations for builds without hardware support */
void sha512_block_native(uint64_t *state, const uint64_t *block) {
(void)state;
(void)block;
diff --git a/lib/Crypto/Hash/SHA512.hs b/lib/Crypto/Hash/SHA512.hs
@@ -12,7 +12,7 @@
-- [2104](https://datatracker.ietf.org/doc/html/rfc2104).
--
-- The 'hash' and 'hmac' functions will use primitive instructions from
--- the ARM cryptographic extensions via FFI if they're available, and
+-- the ARM or x86 SHA512 extensions via FFI if they're available, and
-- will otherwise use a pure Haskell implementation.
module Crypto.Hash.SHA512 (
diff --git a/lib/Crypto/Hash/SHA512/Native.hs b/lib/Crypto/Hash/SHA512/Native.hs
@@ -10,7 +10,8 @@
-- License: MIT
-- Maintainer: Jared Tobin <jared@ppad.tech>
--
--- ARM crypto extension support for SHA-512.
+-- Hardware-accelerated SHA-512, via the ARM or x86 SHA512
+-- extensions.
module Crypto.Hash.SHA512.Native (
sha512_native_available
diff --git a/lib/Crypto/Hash/SHA512/Pure.hs b/lib/Crypto/Hash/SHA512/Pure.hs
@@ -10,7 +10,7 @@
-- Maintainer: Jared Tobin <jared@ppad.tech>
--
-- Pure Haskell SHA-512 and HMAC-SHA512 implementations for strict
--- ByteStrings, used when the ARM cryptographic extensions are
+-- ByteStrings, used when hardware SHA-512 instructions are
-- unavailable.
module Crypto.Hash.SHA512.Pure (
