commit c36cd59073c97a3cde999102ffd5dca7ccd7c02f
parent 254e76a830d56e383fada03a588fdb9e919f46a9
Author: Jared Tobin <jared@jtobin.io>
Date: Sat, 31 Oct 2026 14:29:49 +0400
cbits: add an x86 SSE2 kernel
On x86-64, chacha20_block_native and chacha20_cipher_native now use
SSE2, with the same intra-block layout as the NEON kernel: one
register per row of the state matrix, so that a column round is one
set of element-wise operations, and PSHUFD to rotate rows into and out
of diagonal position. Rotate-by-16 swaps 16-bit halves with
PSHUFLW/PSHUFHW; the other rotations are shift-shift-or pairs.
SSE2 is part of the x86-64 baseline, so chacha20_native_available
returns 1 there without a runtime check. The constants row is now
shared between the ARM and x86 kernels, and the stubs only serve other
architectures.
Diffstat:
2 files changed, 177 insertions(+), 8 deletions(-)
diff --git a/cbits/chacha20_native.c b/cbits/chacha20_native.c
@@ -6,6 +6,22 @@
#include <arm_neon.h>
+#elif defined(__x86_64__)
+
+#include <emmintrin.h>
+
+#endif
+
+#if defined(__aarch64__) || defined(__x86_64__)
+
+static const uint32_t chacha_constants[4] = {
+ 0x61707865u, 0x3320646eu, 0x79622d32u, 0x6b206574u
+};
+
+#endif
+
+#if defined(__aarch64__)
+
/*
* ChaCha20 NEON kernel using intra-block parallelism. The 16-word
* state matrix
@@ -72,10 +88,6 @@ static inline void chacha20_core(uint32x4_t *v0, uint32x4_t *v1,
*v3 = vaddq_u32(d, s3);
}
-static const uint32_t chacha_constants[4] = {
- 0x61707865u, 0x3320646eu, 0x79622d32u, 0x6b206574u
-};
-
/* Set up the constant rows of the state from key + nonce. s3
* (counter + nonce) varies per block and is built inside the loop. */
static inline void chacha20_setup(const uint8_t key[32],
@@ -174,10 +186,167 @@ int chacha20_native_available(void) {
return 1;
}
+#elif defined(__x86_64__)
+
+/*
+ * ChaCha20 SSE2 kernel, using the same intra-block layout as the NEON
+ * kernel: the state matrix is held in four 128-bit registers, one per
+ * row, so that a column round is one set of element-wise vector
+ * operations. Diagonal rounds are reached by left-rotating rows 1, 2
+ * and 3 by 1, 2 and 3 lanes respectively with PSHUFD, then rotating
+ * back. SSE2 is part of the x86-64 baseline, so no runtime check is
+ * needed.
+ */
+
+/* 32-bit left rotations. Rotate-by-16 swaps the 16-bit halves of
+ * each lane with PSHUFLW/PSHUFHW; the others are a shift-shift-or
+ * pair. */
+#define ROTL32_16(x) \
+ _mm_shufflehi_epi16(_mm_shufflelo_epi16((x), 0xb1), 0xb1)
+#define ROTL32_N(x, n) \
+ _mm_or_si128(_mm_slli_epi32((x), (n)), _mm_srli_epi32((x), 32 - (n)))
+
+#define QUARTER(v0, v1, v2, v3) \
+ do { \
+ v0 = _mm_add_epi32(v0, v1); \
+ v3 = _mm_xor_si128(v3, v0); v3 = ROTL32_16(v3); \
+ v2 = _mm_add_epi32(v2, v3); \
+ v1 = _mm_xor_si128(v1, v2); v1 = ROTL32_N(v1, 12); \
+ v0 = _mm_add_epi32(v0, v1); \
+ v3 = _mm_xor_si128(v3, v0); v3 = ROTL32_N(v3, 8); \
+ v2 = _mm_add_epi32(v2, v3); \
+ v1 = _mm_xor_si128(v1, v2); v1 = ROTL32_N(v1, 7); \
+ } while (0)
+
+/* 20-round ChaCha20 core: 10 iterations of (column + diagonal). */
+static inline void chacha20_core(__m128i *v0, __m128i *v1,
+ __m128i *v2, __m128i *v3,
+ __m128i s0, __m128i s1,
+ __m128i s2, __m128i s3) {
+ __m128i a = s0, b = s1, c = s2, d = s3;
+ for (int i = 0; i < 10; i++) {
+ QUARTER(a, b, c, d);
+ /* shift rows: row 1 left 1, row 2 left 2, row 3 left 3. */
+ b = _mm_shuffle_epi32(b, _MM_SHUFFLE(0, 3, 2, 1));
+ c = _mm_shuffle_epi32(c, _MM_SHUFFLE(1, 0, 3, 2));
+ d = _mm_shuffle_epi32(d, _MM_SHUFFLE(2, 1, 0, 3));
+ QUARTER(a, b, c, d);
+ /* shift back. */
+ b = _mm_shuffle_epi32(b, _MM_SHUFFLE(2, 1, 0, 3));
+ c = _mm_shuffle_epi32(c, _MM_SHUFFLE(1, 0, 3, 2));
+ d = _mm_shuffle_epi32(d, _MM_SHUFFLE(0, 3, 2, 1));
+ }
+ *v0 = _mm_add_epi32(a, s0);
+ *v1 = _mm_add_epi32(b, s1);
+ *v2 = _mm_add_epi32(c, s2);
+ *v3 = _mm_add_epi32(d, s3);
+}
+
+/* Set up the constant rows of the state from key + nonce. s3
+ * (counter + nonce) varies per block and is built inside the loop. */
+static inline void chacha20_setup(const uint8_t key[32],
+ const uint8_t nonce[12],
+ __m128i *s0, __m128i *s1,
+ __m128i *s2,
+ uint32_t *n0, uint32_t *n1,
+ uint32_t *n2) {
+ *s0 = _mm_loadu_si128((const __m128i *)chacha_constants);
+ *s1 = _mm_loadu_si128((const __m128i *)key);
+ *s2 = _mm_loadu_si128((const __m128i *)(key + 16));
+ memcpy(n0, nonce + 0, 4);
+ memcpy(n1, nonce + 4, 4);
+ memcpy(n2, nonce + 8, 4);
+}
+
+/* Build row 3 (counter + nonce) for one block. */
+static inline __m128i chacha20_row3(uint32_t counter, uint32_t n0,
+ uint32_t n1, uint32_t n2) {
+ return _mm_setr_epi32((int)counter, (int)n0, (int)n1, (int)n2);
+}
+
+/*
+ * Generate one 64-byte ChaCha20 keystream block at 'out'.
+ */
+void chacha20_block_native(const uint8_t key[32], uint32_t counter,
+ const uint8_t nonce[12], uint8_t out[64]) {
+ __m128i s0, s1, s2;
+ uint32_t n0, n1, n2;
+ chacha20_setup(key, nonce, &s0, &s1, &s2, &n0, &n1, &n2);
+
+ __m128i s3 = chacha20_row3(counter, n0, n1, n2);
+ __m128i v0, v1, v2, v3;
+ chacha20_core(&v0, &v1, &v2, &v3, s0, s1, s2, s3);
+
+ _mm_storeu_si128((__m128i *)(out + 0), v0);
+ _mm_storeu_si128((__m128i *)(out + 16), v1);
+ _mm_storeu_si128((__m128i *)(out + 32), v2);
+ _mm_storeu_si128((__m128i *)(out + 48), v3);
+}
+
+/*
+ * Encrypt/decrypt 'inlen' bytes at 'in' into 'out' using ChaCha20
+ * with the given key, starting counter, and nonce. Stream cipher,
+ * so the same routine decrypts.
+ */
+void chacha20_cipher_native(const uint8_t key[32], uint32_t counter,
+ const uint8_t nonce[12],
+ const uint8_t *in, uint8_t *out,
+ size_t inlen) {
+ __m128i s0, s1, s2;
+ uint32_t n0, n1, n2;
+ chacha20_setup(key, nonce, &s0, &s1, &s2, &n0, &n1, &n2);
+
+ size_t pos = 0;
+ while (pos + 64 <= inlen) {
+ __m128i s3 = chacha20_row3(counter, n0, n1, n2);
+ __m128i v0, v1, v2, v3;
+ chacha20_core(&v0, &v1, &v2, &v3, s0, s1, s2, s3);
+
+ __m128i i0 = _mm_loadu_si128((const __m128i *)(in + pos + 0));
+ __m128i i1 = _mm_loadu_si128((const __m128i *)(in + pos + 16));
+ __m128i i2 = _mm_loadu_si128((const __m128i *)(in + pos + 32));
+ __m128i i3 = _mm_loadu_si128((const __m128i *)(in + pos + 48));
+
+ _mm_storeu_si128((__m128i *)(out + pos + 0),
+ _mm_xor_si128(i0, v0));
+ _mm_storeu_si128((__m128i *)(out + pos + 16),
+ _mm_xor_si128(i1, v1));
+ _mm_storeu_si128((__m128i *)(out + pos + 32),
+ _mm_xor_si128(i2, v2));
+ _mm_storeu_si128((__m128i *)(out + pos + 48),
+ _mm_xor_si128(i3, v3));
+
+ pos += 64;
+ counter++;
+ }
+
+ /* trailing partial block (< 64 bytes) */
+ if (pos < inlen) {
+ __m128i s3 = chacha20_row3(counter, n0, n1, n2);
+ __m128i v0, v1, v2, v3;
+ chacha20_core(&v0, &v1, &v2, &v3, s0, s1, s2, s3);
+
+ uint8_t block[64];
+ _mm_storeu_si128((__m128i *)(block + 0), v0);
+ _mm_storeu_si128((__m128i *)(block + 16), v1);
+ _mm_storeu_si128((__m128i *)(block + 32), v2);
+ _mm_storeu_si128((__m128i *)(block + 48), v3);
+
+ size_t remaining = inlen - pos;
+ for (size_t i = 0; i < remaining; i++) {
+ out[pos + i] = in[pos + i] ^ block[i];
+ }
+ }
+}
+
+int chacha20_native_available(void) {
+ return 1;
+}
+
#else
-/* stubs for non-aarch64 builds; never reached because dispatch is
- * gated on 'chacha20_native_available' returning 0 */
+/* stubs for other architectures; never reached because dispatch is
+ * gated on 'chacha20_native_available' returning 0 */
void chacha20_block_native(const uint8_t *key, uint32_t counter,
const uint8_t *nonce, uint8_t *out) {
diff --git a/lib/Crypto/Cipher/ChaCha20/Native.hs b/lib/Crypto/Cipher/ChaCha20/Native.hs
@@ -7,7 +7,7 @@
-- License: MIT
-- Maintainer: Jared Tobin <jared@ppad.tech>
--
--- ARM NEON support for the ChaCha20 stream cipher.
+-- Vectorised ChaCha20, via ARM NEON or x86 SSE2.
module Crypto.Cipher.ChaCha20.Native (
chacha20_native_available
@@ -45,7 +45,7 @@ fi = fromIntegral
-- api ------------------------------------------------------------------------
--- | Are ARM NEON extensions available?
+-- | Is a vectorised kernel available?
chacha20_native_available :: Bool
chacha20_native_available =
unsafeDupablePerformIO c_chacha20_native_available /= 0
