chacha20_native.c (23171B)
1 #include <stddef.h> 2 #include <stdint.h> 3 #include <string.h> 4 5 #if defined(__aarch64__) 6 7 #include <arm_neon.h> 8 9 #elif defined(__x86_64__) 10 11 #include <cpuid.h> 12 #include <immintrin.h> 13 14 #endif 15 16 #if defined(__aarch64__) || defined(__x86_64__) 17 18 static const uint32_t chacha_constants[4] = { 19 0x61707865u, 0x3320646eu, 0x79622d32u, 0x6b206574u 20 }; 21 22 #endif 23 24 #if defined(__aarch64__) 25 26 /* 27 * ChaCha20 NEON kernel using intra-block parallelism. The 16-word 28 * state matrix 29 * 30 * s00 s01 s02 s03 31 * s04 s05 s06 s07 32 * s08 s09 s10 s11 33 * s12 s13 s14 s15 34 * 35 * is held in four 128-bit NEON registers v0..v3, one per row. A 36 * column quarter-round on (s00, s04, s08, s12), (s01, s05, s09, s13), 37 * etc., becomes one set of element-wise vector operations on 38 * (v0, v1, v2, v3) — four quarter-rounds in parallel. Diagonal 39 * rounds are reached by left-rotating v1, v2, v3 by 1, 2, 3 lanes 40 * respectively with VEXT before the second quarter-round, then 41 * rotating back. 42 */ 43 44 /* 32-bit left rotations. Rotate-by-16 reduces to REV32.u16; the 45 * others compile to a shift-shift-or pair (the compiler folds rotate- 46 * by-8 to a TBL with a constant shuffle on some targets). */ 47 #define ROTL32_16(x) \ 48 vreinterpretq_u32_u16(vrev32q_u16(vreinterpretq_u16_u32(x))) 49 #define ROTL32_12(x) \ 50 vorrq_u32(vshlq_n_u32((x), 12), vshrq_n_u32((x), 20)) 51 #define ROTL32_8(x) \ 52 vorrq_u32(vshlq_n_u32((x), 8), vshrq_n_u32((x), 24)) 53 #define ROTL32_7(x) \ 54 vorrq_u32(vshlq_n_u32((x), 7), vshrq_n_u32((x), 25)) 55 56 #define QUARTER(v0, v1, v2, v3) \ 57 do { \ 58 v0 = vaddq_u32(v0, v1); \ 59 v3 = veorq_u32(v3, v0); v3 = ROTL32_16(v3); \ 60 v2 = vaddq_u32(v2, v3); \ 61 v1 = veorq_u32(v1, v2); v1 = ROTL32_12(v1); \ 62 v0 = vaddq_u32(v0, v1); \ 63 v3 = veorq_u32(v3, v0); v3 = ROTL32_8(v3); \ 64 v2 = vaddq_u32(v2, v3); \ 65 v1 = veorq_u32(v1, v2); v1 = ROTL32_7(v1); \ 66 } while (0) 67 68 /* 20-round ChaCha20 core: 10 iterations of (column + diagonal). */ 69 static inline void chacha20_core(uint32x4_t *v0, uint32x4_t *v1, 70 uint32x4_t *v2, uint32x4_t *v3, 71 uint32x4_t s0, uint32x4_t s1, 72 uint32x4_t s2, uint32x4_t s3) { 73 uint32x4_t a = s0, b = s1, c = s2, d = s3; 74 for (int i = 0; i < 10; i++) { 75 QUARTER(a, b, c, d); 76 /* shift rows: row 1 left 1, row 2 left 2, row 3 left 3. */ 77 b = vextq_u32(b, b, 1); 78 c = vextq_u32(c, c, 2); 79 d = vextq_u32(d, d, 3); 80 QUARTER(a, b, c, d); 81 /* shift back. */ 82 b = vextq_u32(b, b, 3); 83 c = vextq_u32(c, c, 2); 84 d = vextq_u32(d, d, 1); 85 } 86 *v0 = vaddq_u32(a, s0); 87 *v1 = vaddq_u32(b, s1); 88 *v2 = vaddq_u32(c, s2); 89 *v3 = vaddq_u32(d, s3); 90 } 91 92 /* Set up the constant rows of the state from key + nonce. s3 93 * (counter + nonce) varies per block and is built inside the loop. */ 94 static inline void chacha20_setup(const uint8_t key[32], 95 const uint8_t nonce[12], 96 uint32x4_t *s0, uint32x4_t *s1, 97 uint32x4_t *s2, 98 uint32_t *n0, uint32_t *n1, 99 uint32_t *n2) { 100 *s0 = vld1q_u32(chacha_constants); 101 *s1 = vreinterpretq_u32_u8(vld1q_u8(key)); 102 *s2 = vreinterpretq_u32_u8(vld1q_u8(key + 16)); 103 memcpy(n0, nonce + 0, 4); 104 memcpy(n1, nonce + 4, 4); 105 memcpy(n2, nonce + 8, 4); 106 } 107 108 /* 109 * Generate one 64-byte ChaCha20 keystream block at 'out'. 110 */ 111 void chacha20_block_native(const uint8_t key[32], uint32_t counter, 112 const uint8_t nonce[12], uint8_t out[64]) { 113 uint32x4_t s0, s1, s2; 114 uint32_t n0, n1, n2; 115 chacha20_setup(key, nonce, &s0, &s1, &s2, &n0, &n1, &n2); 116 117 uint32_t s3_in[4] = { counter, n0, n1, n2 }; 118 uint32x4_t s3 = vld1q_u32(s3_in); 119 uint32x4_t v0, v1, v2, v3; 120 chacha20_core(&v0, &v1, &v2, &v3, s0, s1, s2, s3); 121 122 vst1q_u8(out + 0, vreinterpretq_u8_u32(v0)); 123 vst1q_u8(out + 16, vreinterpretq_u8_u32(v1)); 124 vst1q_u8(out + 32, vreinterpretq_u8_u32(v2)); 125 vst1q_u8(out + 48, vreinterpretq_u8_u32(v3)); 126 } 127 128 /* 129 * Encrypt/decrypt 'inlen' bytes at 'in' into 'out' using ChaCha20 130 * with the given key, starting counter, and nonce. Stream cipher, 131 * so the same routine decrypts. 132 */ 133 void chacha20_cipher_native(const uint8_t key[32], uint32_t counter, 134 const uint8_t nonce[12], 135 const uint8_t *in, uint8_t *out, 136 size_t inlen) { 137 uint32x4_t s0, s1, s2; 138 uint32_t n0, n1, n2; 139 chacha20_setup(key, nonce, &s0, &s1, &s2, &n0, &n1, &n2); 140 141 size_t pos = 0; 142 while (pos + 64 <= inlen) { 143 uint32_t s3_in[4] = { counter, n0, n1, n2 }; 144 uint32x4_t s3 = vld1q_u32(s3_in); 145 uint32x4_t v0, v1, v2, v3; 146 chacha20_core(&v0, &v1, &v2, &v3, s0, s1, s2, s3); 147 148 uint8x16_t i0 = vld1q_u8(in + pos + 0); 149 uint8x16_t i1 = vld1q_u8(in + pos + 16); 150 uint8x16_t i2 = vld1q_u8(in + pos + 32); 151 uint8x16_t i3 = vld1q_u8(in + pos + 48); 152 153 vst1q_u8(out + pos + 0, 154 veorq_u8(i0, vreinterpretq_u8_u32(v0))); 155 vst1q_u8(out + pos + 16, 156 veorq_u8(i1, vreinterpretq_u8_u32(v1))); 157 vst1q_u8(out + pos + 32, 158 veorq_u8(i2, vreinterpretq_u8_u32(v2))); 159 vst1q_u8(out + pos + 48, 160 veorq_u8(i3, vreinterpretq_u8_u32(v3))); 161 162 pos += 64; 163 counter++; 164 } 165 166 /* trailing partial block (< 64 bytes) */ 167 if (pos < inlen) { 168 uint32_t s3_in[4] = { counter, n0, n1, n2 }; 169 uint32x4_t s3 = vld1q_u32(s3_in); 170 uint32x4_t v0, v1, v2, v3; 171 chacha20_core(&v0, &v1, &v2, &v3, s0, s1, s2, s3); 172 173 uint8_t block[64]; 174 vst1q_u8(block + 0, vreinterpretq_u8_u32(v0)); 175 vst1q_u8(block + 16, vreinterpretq_u8_u32(v1)); 176 vst1q_u8(block + 32, vreinterpretq_u8_u32(v2)); 177 vst1q_u8(block + 48, vreinterpretq_u8_u32(v3)); 178 179 size_t remaining = inlen - pos; 180 for (size_t i = 0; i < remaining; i++) { 181 out[pos + i] = in[pos + i] ^ block[i]; 182 } 183 } 184 } 185 186 int chacha20_native_available(void) { 187 return 1; 188 } 189 190 #elif defined(__x86_64__) 191 192 /* 193 * ChaCha20 SSE2 kernel, using the same intra-block layout as the NEON 194 * kernel: the state matrix is held in four 128-bit registers, one per 195 * row, so that a column round is one set of element-wise vector 196 * operations. Diagonal rounds are reached by left-rotating rows 1, 2 197 * and 3 by 1, 2 and 3 lanes respectively with PSHUFD, then rotating 198 * back. SSE2 is part of the x86-64 baseline, so no runtime check is 199 * needed. 200 */ 201 202 /* 32-bit left rotations. Rotate-by-16 swaps the 16-bit halves of 203 * each lane with PSHUFLW/PSHUFHW; the others are a shift-shift-or 204 * pair. */ 205 #define ROTL32_16(x) \ 206 _mm_shufflehi_epi16(_mm_shufflelo_epi16((x), 0xb1), 0xb1) 207 #define ROTL32_N(x, n) \ 208 _mm_or_si128(_mm_slli_epi32((x), (n)), _mm_srli_epi32((x), 32 - (n))) 209 210 #define QUARTER(v0, v1, v2, v3) \ 211 do { \ 212 v0 = _mm_add_epi32(v0, v1); \ 213 v3 = _mm_xor_si128(v3, v0); v3 = ROTL32_16(v3); \ 214 v2 = _mm_add_epi32(v2, v3); \ 215 v1 = _mm_xor_si128(v1, v2); v1 = ROTL32_N(v1, 12); \ 216 v0 = _mm_add_epi32(v0, v1); \ 217 v3 = _mm_xor_si128(v3, v0); v3 = ROTL32_N(v3, 8); \ 218 v2 = _mm_add_epi32(v2, v3); \ 219 v1 = _mm_xor_si128(v1, v2); v1 = ROTL32_N(v1, 7); \ 220 } while (0) 221 222 /* 20-round ChaCha20 core: 10 iterations of (column + diagonal). */ 223 static inline void chacha20_core(__m128i *v0, __m128i *v1, 224 __m128i *v2, __m128i *v3, 225 __m128i s0, __m128i s1, 226 __m128i s2, __m128i s3) { 227 __m128i a = s0, b = s1, c = s2, d = s3; 228 for (int i = 0; i < 10; i++) { 229 QUARTER(a, b, c, d); 230 /* shift rows: row 1 left 1, row 2 left 2, row 3 left 3. */ 231 b = _mm_shuffle_epi32(b, _MM_SHUFFLE(0, 3, 2, 1)); 232 c = _mm_shuffle_epi32(c, _MM_SHUFFLE(1, 0, 3, 2)); 233 d = _mm_shuffle_epi32(d, _MM_SHUFFLE(2, 1, 0, 3)); 234 QUARTER(a, b, c, d); 235 /* shift back. */ 236 b = _mm_shuffle_epi32(b, _MM_SHUFFLE(2, 1, 0, 3)); 237 c = _mm_shuffle_epi32(c, _MM_SHUFFLE(1, 0, 3, 2)); 238 d = _mm_shuffle_epi32(d, _MM_SHUFFLE(0, 3, 2, 1)); 239 } 240 *v0 = _mm_add_epi32(a, s0); 241 *v1 = _mm_add_epi32(b, s1); 242 *v2 = _mm_add_epi32(c, s2); 243 *v3 = _mm_add_epi32(d, s3); 244 } 245 246 /* Set up the constant rows of the state from key + nonce. s3 247 * (counter + nonce) varies per block and is built inside the loop. */ 248 static inline void chacha20_setup(const uint8_t key[32], 249 const uint8_t nonce[12], 250 __m128i *s0, __m128i *s1, 251 __m128i *s2, 252 uint32_t *n0, uint32_t *n1, 253 uint32_t *n2) { 254 *s0 = _mm_loadu_si128((const __m128i *)chacha_constants); 255 *s1 = _mm_loadu_si128((const __m128i *)key); 256 *s2 = _mm_loadu_si128((const __m128i *)(key + 16)); 257 memcpy(n0, nonce + 0, 4); 258 memcpy(n1, nonce + 4, 4); 259 memcpy(n2, nonce + 8, 4); 260 } 261 262 /* Build row 3 (counter + nonce) for one block. */ 263 static inline __m128i chacha20_row3(uint32_t counter, uint32_t n0, 264 uint32_t n1, uint32_t n2) { 265 return _mm_setr_epi32((int)counter, (int)n0, (int)n1, (int)n2); 266 } 267 268 /* 269 * ChaCha20 AVX2 kernel: two blocks at once, in the same row layout. 270 * Each 256-bit register holds one row of the state for two 271 * consecutive blocks, the low 128-bit lane for block n and the high 272 * lane for block n + 1. VPSHUFD and VPSHUFB act within each lane, so 273 * the rounds are those of the SSE2 kernel on wider registers; only 274 * row 3, whose counters differ, needs per-lane setup. 275 * 276 * Each round is one long dependency chain, so long inputs are 277 * processed four blocks at a time, as two interleaved pairs, to keep 278 * more of the vector units busy. 279 * 280 * AVX2 is not part of the x86-64 baseline, so the kernel is compiled 281 * for it alone and selected at runtime by 'have_avx2'. 282 */ 283 284 /* 285 * Return 1 if the CPU implements AVX2 and the OS saves and restores 286 * the ymm registers, 0 otherwise. 287 */ 288 static int avx2_detect(void) { 289 unsigned int eax, ebx, ecx, edx; 290 if (!__get_cpuid(1, &eax, &ebx, &ecx, &edx)) 291 return 0; 292 if (!(ecx & bit_OSXSAVE) || !(ecx & bit_AVX)) 293 return 0; 294 /* XCR0 bits 1 and 2: xmm and ymm state enabled by the OS */ 295 unsigned int xcr0_lo, xcr0_hi; 296 __asm__ volatile ("xgetbv" : "=a"(xcr0_lo), "=d"(xcr0_hi) : "c"(0)); 297 if ((xcr0_lo & 6) != 6) 298 return 0; 299 if (!__get_cpuid_count(7, 0, &eax, &ebx, &ecx, &edx)) 300 return 0; 301 return (ebx & bit_AVX2) != 0; 302 } 303 304 /* 'avx2_detect', cached; CPUID can be slow, especially under a 305 * hypervisor. Concurrent first calls compute the same value. */ 306 static int have_avx2(void) { 307 static int cached = -1; 308 int v = __atomic_load_n(&cached, __ATOMIC_RELAXED); 309 if (v < 0) { 310 v = avx2_detect(); 311 __atomic_store_n(&cached, v, __ATOMIC_RELAXED); 312 } 313 return v; 314 } 315 316 /* 32-bit left rotations. Rotate-by-16 and rotate-by-8 move whole 317 * bytes, so are single VPSHUFB byte permutations. */ 318 __attribute__((target("avx2"))) 319 static inline __m256i rotl32_16_avx2(__m256i x) { 320 const __m256i idx = _mm256_setr_epi8( 321 2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13, 322 2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13); 323 return _mm256_shuffle_epi8(x, idx); 324 } 325 326 __attribute__((target("avx2"))) 327 static inline __m256i rotl32_8_avx2(__m256i x) { 328 const __m256i idx = _mm256_setr_epi8( 329 3, 0, 1, 2, 7, 4, 5, 6, 11, 8, 9, 10, 15, 12, 13, 14, 330 3, 0, 1, 2, 7, 4, 5, 6, 11, 8, 9, 10, 15, 12, 13, 14); 331 return _mm256_shuffle_epi8(x, idx); 332 } 333 334 #define ROTL32_N_AVX2(x, n) \ 335 _mm256_or_si256(_mm256_slli_epi32((x), (n)), \ 336 _mm256_srli_epi32((x), 32 - (n))) 337 338 #define QUARTER_AVX2(v0, v1, v2, v3) \ 339 do { \ 340 v0 = _mm256_add_epi32(v0, v1); \ 341 v3 = _mm256_xor_si256(v3, v0); v3 = rotl32_16_avx2(v3); \ 342 v2 = _mm256_add_epi32(v2, v3); \ 343 v1 = _mm256_xor_si256(v1, v2); v1 = ROTL32_N_AVX2(v1, 12); \ 344 v0 = _mm256_add_epi32(v0, v1); \ 345 v3 = _mm256_xor_si256(v3, v0); v3 = rotl32_8_avx2(v3); \ 346 v2 = _mm256_add_epi32(v2, v3); \ 347 v1 = _mm256_xor_si256(v1, v2); v1 = ROTL32_N_AVX2(v1, 7); \ 348 } while (0) 349 350 /* 20-round ChaCha20 core over two blocks. */ 351 __attribute__((target("avx2"))) 352 static inline void chacha20_core_avx2(__m256i *v0, __m256i *v1, 353 __m256i *v2, __m256i *v3, 354 __m256i s0, __m256i s1, 355 __m256i s2, __m256i s3) { 356 __m256i a = s0, b = s1, c = s2, d = s3; 357 for (int i = 0; i < 10; i++) { 358 QUARTER_AVX2(a, b, c, d); 359 b = _mm256_shuffle_epi32(b, _MM_SHUFFLE(0, 3, 2, 1)); 360 c = _mm256_shuffle_epi32(c, _MM_SHUFFLE(1, 0, 3, 2)); 361 d = _mm256_shuffle_epi32(d, _MM_SHUFFLE(2, 1, 0, 3)); 362 QUARTER_AVX2(a, b, c, d); 363 b = _mm256_shuffle_epi32(b, _MM_SHUFFLE(2, 1, 0, 3)); 364 c = _mm256_shuffle_epi32(c, _MM_SHUFFLE(1, 0, 3, 2)); 365 d = _mm256_shuffle_epi32(d, _MM_SHUFFLE(0, 3, 2, 1)); 366 } 367 *v0 = _mm256_add_epi32(a, s0); 368 *v1 = _mm256_add_epi32(b, s1); 369 *v2 = _mm256_add_epi32(c, s2); 370 *v3 = _mm256_add_epi32(d, s3); 371 } 372 373 /* 20-round ChaCha20 core over four blocks, as two interleaved 374 * pairs so that their independent dependency chains overlap. */ 375 __attribute__((target("avx2"))) 376 static inline void chacha20_core4_avx2(__m256i v[8], const __m256i s[8]) { 377 __m256i a = s[0], b = s[1], c = s[2], d = s[3]; 378 __m256i e = s[4], f = s[5], g = s[6], h = s[7]; 379 for (int i = 0; i < 10; i++) { 380 QUARTER_AVX2(a, b, c, d); 381 QUARTER_AVX2(e, f, g, h); 382 b = _mm256_shuffle_epi32(b, _MM_SHUFFLE(0, 3, 2, 1)); 383 c = _mm256_shuffle_epi32(c, _MM_SHUFFLE(1, 0, 3, 2)); 384 d = _mm256_shuffle_epi32(d, _MM_SHUFFLE(2, 1, 0, 3)); 385 f = _mm256_shuffle_epi32(f, _MM_SHUFFLE(0, 3, 2, 1)); 386 g = _mm256_shuffle_epi32(g, _MM_SHUFFLE(1, 0, 3, 2)); 387 h = _mm256_shuffle_epi32(h, _MM_SHUFFLE(2, 1, 0, 3)); 388 QUARTER_AVX2(a, b, c, d); 389 QUARTER_AVX2(e, f, g, h); 390 b = _mm256_shuffle_epi32(b, _MM_SHUFFLE(2, 1, 0, 3)); 391 c = _mm256_shuffle_epi32(c, _MM_SHUFFLE(1, 0, 3, 2)); 392 d = _mm256_shuffle_epi32(d, _MM_SHUFFLE(0, 3, 2, 1)); 393 f = _mm256_shuffle_epi32(f, _MM_SHUFFLE(2, 1, 0, 3)); 394 g = _mm256_shuffle_epi32(g, _MM_SHUFFLE(1, 0, 3, 2)); 395 h = _mm256_shuffle_epi32(h, _MM_SHUFFLE(0, 3, 2, 1)); 396 } 397 v[0] = _mm256_add_epi32(a, s[0]); 398 v[1] = _mm256_add_epi32(b, s[1]); 399 v[2] = _mm256_add_epi32(c, s[2]); 400 v[3] = _mm256_add_epi32(d, s[3]); 401 v[4] = _mm256_add_epi32(e, s[4]); 402 v[5] = _mm256_add_epi32(f, s[5]); 403 v[6] = _mm256_add_epi32(g, s[6]); 404 v[7] = _mm256_add_epi32(h, s[7]); 405 } 406 407 /* XOR one block pair's keystream rows into 128 bytes of input. */ 408 __attribute__((target("avx2"))) 409 static inline void chacha20_xor2_avx2(const uint8_t *in, uint8_t *out, 410 __m256i v0, __m256i v1, 411 __m256i v2, __m256i v3) { 412 /* gather each block's rows: block n from the low lanes, 413 * block n + 1 from the high lanes */ 414 __m256i k0 = _mm256_permute2x128_si256(v0, v1, 0x20); 415 __m256i k1 = _mm256_permute2x128_si256(v2, v3, 0x20); 416 __m256i k2 = _mm256_permute2x128_si256(v0, v1, 0x31); 417 __m256i k3 = _mm256_permute2x128_si256(v2, v3, 0x31); 418 419 __m256i i0 = _mm256_loadu_si256((const __m256i *)(in + 0)); 420 __m256i i1 = _mm256_loadu_si256((const __m256i *)(in + 32)); 421 __m256i i2 = _mm256_loadu_si256((const __m256i *)(in + 64)); 422 __m256i i3 = _mm256_loadu_si256((const __m256i *)(in + 96)); 423 424 _mm256_storeu_si256((__m256i *)(out + 0), _mm256_xor_si256(i0, k0)); 425 _mm256_storeu_si256((__m256i *)(out + 32), _mm256_xor_si256(i1, k1)); 426 _mm256_storeu_si256((__m256i *)(out + 64), _mm256_xor_si256(i2, k2)); 427 _mm256_storeu_si256((__m256i *)(out + 96), _mm256_xor_si256(i3, k3)); 428 } 429 430 /* 431 * Encrypt/decrypt the longest prefix of 'in' that is a whole number 432 * of 128-byte block pairs, starting at 'counter': four blocks at a 433 * time while at least 256 bytes remain, then two. Returns the number 434 * of bytes processed. 435 */ 436 __attribute__((target("avx2"))) 437 static size_t chacha20_cipher_avx2(const uint8_t key[32], 438 uint32_t counter, 439 const uint8_t nonce[12], 440 const uint8_t *in, uint8_t *out, 441 size_t inlen) { 442 __m256i s0 = _mm256_broadcastsi128_si256( 443 _mm_loadu_si128((const __m128i *)chacha_constants)); 444 __m256i s1 = _mm256_broadcastsi128_si256( 445 _mm_loadu_si128((const __m128i *)key)); 446 __m256i s2 = _mm256_broadcastsi128_si256( 447 _mm_loadu_si128((const __m128i *)(key + 16))); 448 uint32_t n0, n1, n2; 449 memcpy(&n0, nonce + 0, 4); 450 memcpy(&n1, nonce + 4, 4); 451 memcpy(&n2, nonce + 8, 4); 452 453 size_t pos = 0; 454 while (pos + 256 <= inlen) { 455 uint32_t c1 = counter + 1, c2 = counter + 2, c3 = counter + 3; 456 __m256i s[8] = { 457 s0, s1, s2, 458 _mm256_setr_epi32((int)counter, (int)n0, (int)n1, (int)n2, 459 (int)c1, (int)n0, (int)n1, (int)n2), 460 s0, s1, s2, 461 _mm256_setr_epi32((int)c2, (int)n0, (int)n1, (int)n2, 462 (int)c3, (int)n0, (int)n1, (int)n2) 463 }; 464 __m256i v[8]; 465 chacha20_core4_avx2(v, s); 466 chacha20_xor2_avx2(in + pos, out + pos, v[0], v[1], v[2], v[3]); 467 chacha20_xor2_avx2(in + pos + 128, out + pos + 128, 468 v[4], v[5], v[6], v[7]); 469 pos += 256; 470 counter += 4; 471 } 472 while (pos + 128 <= inlen) { 473 uint32_t c1 = counter + 1; 474 __m256i s3 = _mm256_setr_epi32( 475 (int)counter, (int)n0, (int)n1, (int)n2, 476 (int)c1, (int)n0, (int)n1, (int)n2); 477 __m256i v0, v1, v2, v3; 478 chacha20_core_avx2(&v0, &v1, &v2, &v3, s0, s1, s2, s3); 479 chacha20_xor2_avx2(in + pos, out + pos, v0, v1, v2, v3); 480 pos += 128; 481 counter += 2; 482 } 483 return pos; 484 } 485 486 /* 487 * Generate one 64-byte ChaCha20 keystream block at 'out'. 488 */ 489 void chacha20_block_native(const uint8_t key[32], uint32_t counter, 490 const uint8_t nonce[12], uint8_t out[64]) { 491 __m128i s0, s1, s2; 492 uint32_t n0, n1, n2; 493 chacha20_setup(key, nonce, &s0, &s1, &s2, &n0, &n1, &n2); 494 495 __m128i s3 = chacha20_row3(counter, n0, n1, n2); 496 __m128i v0, v1, v2, v3; 497 chacha20_core(&v0, &v1, &v2, &v3, s0, s1, s2, s3); 498 499 _mm_storeu_si128((__m128i *)(out + 0), v0); 500 _mm_storeu_si128((__m128i *)(out + 16), v1); 501 _mm_storeu_si128((__m128i *)(out + 32), v2); 502 _mm_storeu_si128((__m128i *)(out + 48), v3); 503 } 504 505 /* 506 * Encrypt/decrypt 'inlen' bytes at 'in' into 'out' using ChaCha20 507 * with the given key, starting counter, and nonce. Stream cipher, 508 * so the same routine decrypts. 509 */ 510 void chacha20_cipher_native(const uint8_t key[32], uint32_t counter, 511 const uint8_t nonce[12], 512 const uint8_t *in, uint8_t *out, 513 size_t inlen) { 514 __m128i s0, s1, s2; 515 uint32_t n0, n1, n2; 516 chacha20_setup(key, nonce, &s0, &s1, &s2, &n0, &n1, &n2); 517 518 /* block pairs via AVX2 where available, the rest via SSE2 */ 519 size_t pos = 0; 520 if (inlen >= 128 && have_avx2()) { 521 pos = chacha20_cipher_avx2(key, counter, nonce, in, out, inlen); 522 counter += (uint32_t)(pos / 64); 523 } 524 525 while (pos + 64 <= inlen) { 526 __m128i s3 = chacha20_row3(counter, n0, n1, n2); 527 __m128i v0, v1, v2, v3; 528 chacha20_core(&v0, &v1, &v2, &v3, s0, s1, s2, s3); 529 530 __m128i i0 = _mm_loadu_si128((const __m128i *)(in + pos + 0)); 531 __m128i i1 = _mm_loadu_si128((const __m128i *)(in + pos + 16)); 532 __m128i i2 = _mm_loadu_si128((const __m128i *)(in + pos + 32)); 533 __m128i i3 = _mm_loadu_si128((const __m128i *)(in + pos + 48)); 534 535 _mm_storeu_si128((__m128i *)(out + pos + 0), 536 _mm_xor_si128(i0, v0)); 537 _mm_storeu_si128((__m128i *)(out + pos + 16), 538 _mm_xor_si128(i1, v1)); 539 _mm_storeu_si128((__m128i *)(out + pos + 32), 540 _mm_xor_si128(i2, v2)); 541 _mm_storeu_si128((__m128i *)(out + pos + 48), 542 _mm_xor_si128(i3, v3)); 543 544 pos += 64; 545 counter++; 546 } 547 548 /* trailing partial block (< 64 bytes) */ 549 if (pos < inlen) { 550 __m128i s3 = chacha20_row3(counter, n0, n1, n2); 551 __m128i v0, v1, v2, v3; 552 chacha20_core(&v0, &v1, &v2, &v3, s0, s1, s2, s3); 553 554 uint8_t block[64]; 555 _mm_storeu_si128((__m128i *)(block + 0), v0); 556 _mm_storeu_si128((__m128i *)(block + 16), v1); 557 _mm_storeu_si128((__m128i *)(block + 32), v2); 558 _mm_storeu_si128((__m128i *)(block + 48), v3); 559 560 size_t remaining = inlen - pos; 561 for (size_t i = 0; i < remaining; i++) { 562 out[pos + i] = in[pos + i] ^ block[i]; 563 } 564 } 565 } 566 567 int chacha20_native_available(void) { 568 return 1; 569 } 570 571 #else 572 573 /* stubs for other architectures; never reached because dispatch is 574 * gated on 'chacha20_native_available' returning 0 */ 575 576 void chacha20_block_native(const uint8_t *key, uint32_t counter, 577 const uint8_t *nonce, uint8_t *out) { 578 (void)key; (void)counter; (void)nonce; (void)out; 579 } 580 581 void chacha20_cipher_native(const uint8_t *key, uint32_t counter, 582 const uint8_t *nonce, 583 const uint8_t *in, uint8_t *out, 584 size_t inlen) { 585 (void)key; (void)counter; (void)nonce; 586 (void)in; (void)out; (void)inlen; 587 } 588 589 int chacha20_native_available(void) { 590 return 0; 591 } 592 593 #endif