sha256_native.c (13286B)
1 #include <stdint.h> 2 #include <string.h> 3 4 #if defined(__aarch64__) 5 6 #include <arm_neon.h> 7 8 #if defined(__linux__) 9 #include <sys/auxv.h> 10 #ifndef HWCAP_SHA2 11 #define HWCAP_SHA2 (1UL << 6) 12 #endif 13 #endif 14 15 #elif defined(__x86_64__) 16 17 #include <cpuid.h> 18 #include <immintrin.h> 19 20 #ifndef bit_SHA 21 #define bit_SHA (1 << 29) 22 #endif 23 24 #endif 25 26 #if defined(__aarch64__) || defined(__x86_64__) 27 28 static const uint32_t K[64] = { 29 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 30 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, 31 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 32 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, 33 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 34 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, 35 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 36 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, 37 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 38 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 39 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 40 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, 41 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 42 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, 43 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 44 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 45 }; 46 47 #endif 48 49 #if defined(__aarch64__) 50 51 /* 52 * Process one 64-byte block using ARM SHA256 crypto instructions. 53 * 54 * state: pointer to 8 uint32_t words (a,b,c,d,e,f,g,h) 55 * block: pointer to 16 uint32_t words (already native endian) 56 * 57 * The state is updated in place. 58 * 59 * Compiled for the SHA2 extension only; callers must first check 60 * 'sha256_native_available'. 61 */ 62 __attribute__((target("+sha2"))) 63 void sha256_block_native(uint32_t *state, const uint32_t *block) { 64 /* Load current hash state */ 65 uint32x4_t abcd = vld1q_u32(&state[0]); 66 uint32x4_t efgh = vld1q_u32(&state[4]); 67 68 /* Save original for final addition */ 69 uint32x4_t abcd_orig = abcd; 70 uint32x4_t efgh_orig = efgh; 71 72 /* Load message (already native endian) */ 73 uint32x4_t m0 = vld1q_u32(&block[0]); 74 uint32x4_t m1 = vld1q_u32(&block[4]); 75 uint32x4_t m2 = vld1q_u32(&block[8]); 76 uint32x4_t m3 = vld1q_u32(&block[12]); 77 78 uint32x4_t tmp, tmp2; 79 80 /* Rounds 0-3 */ 81 tmp = vaddq_u32(m0, vld1q_u32(&K[0])); 82 tmp2 = abcd; 83 abcd = vsha256hq_u32(abcd, efgh, tmp); 84 efgh = vsha256h2q_u32(efgh, tmp2, tmp); 85 m0 = vsha256su1q_u32(vsha256su0q_u32(m0, m1), m2, m3); 86 87 /* Rounds 4-7 */ 88 tmp = vaddq_u32(m1, vld1q_u32(&K[4])); 89 tmp2 = abcd; 90 abcd = vsha256hq_u32(abcd, efgh, tmp); 91 efgh = vsha256h2q_u32(efgh, tmp2, tmp); 92 m1 = vsha256su1q_u32(vsha256su0q_u32(m1, m2), m3, m0); 93 94 /* Rounds 8-11 */ 95 tmp = vaddq_u32(m2, vld1q_u32(&K[8])); 96 tmp2 = abcd; 97 abcd = vsha256hq_u32(abcd, efgh, tmp); 98 efgh = vsha256h2q_u32(efgh, tmp2, tmp); 99 m2 = vsha256su1q_u32(vsha256su0q_u32(m2, m3), m0, m1); 100 101 /* Rounds 12-15 */ 102 tmp = vaddq_u32(m3, vld1q_u32(&K[12])); 103 tmp2 = abcd; 104 abcd = vsha256hq_u32(abcd, efgh, tmp); 105 efgh = vsha256h2q_u32(efgh, tmp2, tmp); 106 m3 = vsha256su1q_u32(vsha256su0q_u32(m3, m0), m1, m2); 107 108 /* Rounds 16-19 */ 109 tmp = vaddq_u32(m0, vld1q_u32(&K[16])); 110 tmp2 = abcd; 111 abcd = vsha256hq_u32(abcd, efgh, tmp); 112 efgh = vsha256h2q_u32(efgh, tmp2, tmp); 113 m0 = vsha256su1q_u32(vsha256su0q_u32(m0, m1), m2, m3); 114 115 /* Rounds 20-23 */ 116 tmp = vaddq_u32(m1, vld1q_u32(&K[20])); 117 tmp2 = abcd; 118 abcd = vsha256hq_u32(abcd, efgh, tmp); 119 efgh = vsha256h2q_u32(efgh, tmp2, tmp); 120 m1 = vsha256su1q_u32(vsha256su0q_u32(m1, m2), m3, m0); 121 122 /* Rounds 24-27 */ 123 tmp = vaddq_u32(m2, vld1q_u32(&K[24])); 124 tmp2 = abcd; 125 abcd = vsha256hq_u32(abcd, efgh, tmp); 126 efgh = vsha256h2q_u32(efgh, tmp2, tmp); 127 m2 = vsha256su1q_u32(vsha256su0q_u32(m2, m3), m0, m1); 128 129 /* Rounds 28-31 */ 130 tmp = vaddq_u32(m3, vld1q_u32(&K[28])); 131 tmp2 = abcd; 132 abcd = vsha256hq_u32(abcd, efgh, tmp); 133 efgh = vsha256h2q_u32(efgh, tmp2, tmp); 134 m3 = vsha256su1q_u32(vsha256su0q_u32(m3, m0), m1, m2); 135 136 /* Rounds 32-35 */ 137 tmp = vaddq_u32(m0, vld1q_u32(&K[32])); 138 tmp2 = abcd; 139 abcd = vsha256hq_u32(abcd, efgh, tmp); 140 efgh = vsha256h2q_u32(efgh, tmp2, tmp); 141 m0 = vsha256su1q_u32(vsha256su0q_u32(m0, m1), m2, m3); 142 143 /* Rounds 36-39 */ 144 tmp = vaddq_u32(m1, vld1q_u32(&K[36])); 145 tmp2 = abcd; 146 abcd = vsha256hq_u32(abcd, efgh, tmp); 147 efgh = vsha256h2q_u32(efgh, tmp2, tmp); 148 m1 = vsha256su1q_u32(vsha256su0q_u32(m1, m2), m3, m0); 149 150 /* Rounds 40-43 */ 151 tmp = vaddq_u32(m2, vld1q_u32(&K[40])); 152 tmp2 = abcd; 153 abcd = vsha256hq_u32(abcd, efgh, tmp); 154 efgh = vsha256h2q_u32(efgh, tmp2, tmp); 155 m2 = vsha256su1q_u32(vsha256su0q_u32(m2, m3), m0, m1); 156 157 /* Rounds 44-47 */ 158 tmp = vaddq_u32(m3, vld1q_u32(&K[44])); 159 tmp2 = abcd; 160 abcd = vsha256hq_u32(abcd, efgh, tmp); 161 efgh = vsha256h2q_u32(efgh, tmp2, tmp); 162 m3 = vsha256su1q_u32(vsha256su0q_u32(m3, m0), m1, m2); 163 164 /* Rounds 48-51 */ 165 tmp = vaddq_u32(m0, vld1q_u32(&K[48])); 166 tmp2 = abcd; 167 abcd = vsha256hq_u32(abcd, efgh, tmp); 168 efgh = vsha256h2q_u32(efgh, tmp2, tmp); 169 170 /* Rounds 52-55 */ 171 tmp = vaddq_u32(m1, vld1q_u32(&K[52])); 172 tmp2 = abcd; 173 abcd = vsha256hq_u32(abcd, efgh, tmp); 174 efgh = vsha256h2q_u32(efgh, tmp2, tmp); 175 176 /* Rounds 56-59 */ 177 tmp = vaddq_u32(m2, vld1q_u32(&K[56])); 178 tmp2 = abcd; 179 abcd = vsha256hq_u32(abcd, efgh, tmp); 180 efgh = vsha256h2q_u32(efgh, tmp2, tmp); 181 182 /* Rounds 60-63 */ 183 tmp = vaddq_u32(m3, vld1q_u32(&K[60])); 184 tmp2 = abcd; 185 abcd = vsha256hq_u32(abcd, efgh, tmp); 186 efgh = vsha256h2q_u32(efgh, tmp2, tmp); 187 188 /* Add original state back */ 189 abcd = vaddq_u32(abcd, abcd_orig); 190 efgh = vaddq_u32(efgh, efgh_orig); 191 192 /* Store result */ 193 vst1q_u32(&state[0], abcd); 194 vst1q_u32(&state[4], efgh); 195 } 196 197 /* 198 * Return 1 if the CPU implements the ARM SHA2 extension, 0 otherwise. 199 * Every Apple arm64 core implements it. Elsewhere we only know how to 200 * ask Linux, and report 0 (selecting the pure Haskell fallback) on 201 * any other OS. 202 */ 203 int sha256_native_available(void) { 204 #if defined(__APPLE__) 205 return 1; 206 #elif defined(__linux__) 207 return (getauxval(AT_HWCAP) & HWCAP_SHA2) != 0; 208 #else 209 return 0; 210 #endif 211 } 212 213 #elif defined(__x86_64__) 214 215 /* 216 * Four rounds. 'sha256rnds2' does two: it takes the (c,d,g,h) and 217 * (a,b,e,f) halves of the state, and two message-plus-constant words in 218 * the low lanes of its third argument, and returns the new (a,b,e,f). 219 * The old (a,b,e,f) is then the new (c,d,g,h), so a second call with 220 * the halves swapped does the next two. 221 */ 222 #define ROUNDS4(abef, cdgh, msg) \ 223 do { \ 224 cdgh = _mm_sha256rnds2_epu32(cdgh, abef, msg); \ 225 msg = _mm_shuffle_epi32(msg, 0x0e); \ 226 abef = _mm_sha256rnds2_epu32(abef, cdgh, msg); \ 227 } while (0) 228 229 /* 230 * Process one 64-byte block using x86 SHA instructions. 231 * 232 * state: pointer to 8 uint32_t words (a,b,c,d,e,f,g,h) 233 * block: pointer to 16 uint32_t words (already native endian) 234 * 235 * The state is updated in place. 236 * 237 * Compiled for the SHA and SSE4.1 extensions only; callers must first 238 * check 'sha256_native_available'. 239 * 240 * Lane orders in comments are written from highest lane to lowest. 241 */ 242 __attribute__((target("sha,sse4.1"))) 243 void sha256_block_native(uint32_t *state, const uint32_t *block) { 244 __m128i abef, cdgh, abef_orig, cdgh_orig; 245 __m128i m0, m1, m2, m3, msg, tmp; 246 247 /* Load current hash state, rearranging it for the SHA instructions */ 248 tmp = _mm_loadu_si128((const __m128i *)&state[0]); /* d c b a */ 249 cdgh = _mm_loadu_si128((const __m128i *)&state[4]); /* h g f e */ 250 tmp = _mm_shuffle_epi32(tmp, 0xb1); /* c d a b */ 251 cdgh = _mm_shuffle_epi32(cdgh, 0x1b); /* e f g h */ 252 abef = _mm_alignr_epi8(tmp, cdgh, 8); /* a b e f */ 253 cdgh = _mm_blend_epi16(cdgh, tmp, 0xf0); /* c d g h */ 254 255 /* Save original for final addition */ 256 abef_orig = abef; 257 cdgh_orig = cdgh; 258 259 /* Load message (already native endian) */ 260 m0 = _mm_loadu_si128((const __m128i *)&block[0]); 261 m1 = _mm_loadu_si128((const __m128i *)&block[4]); 262 m2 = _mm_loadu_si128((const __m128i *)&block[8]); 263 m3 = _mm_loadu_si128((const __m128i *)&block[12]); 264 265 /* Rounds 0-3 */ 266 msg = _mm_add_epi32(m0, _mm_loadu_si128((const __m128i *)&K[0])); 267 ROUNDS4(abef, cdgh, msg); 268 269 /* Rounds 4-7 */ 270 msg = _mm_add_epi32(m1, _mm_loadu_si128((const __m128i *)&K[4])); 271 ROUNDS4(abef, cdgh, msg); 272 m0 = _mm_sha256msg1_epu32(m0, m1); 273 274 /* Rounds 8-11 */ 275 msg = _mm_add_epi32(m2, _mm_loadu_si128((const __m128i *)&K[8])); 276 ROUNDS4(abef, cdgh, msg); 277 m1 = _mm_sha256msg1_epu32(m1, m2); 278 279 /* Rounds 12-15 */ 280 msg = _mm_add_epi32(m3, _mm_loadu_si128((const __m128i *)&K[12])); 281 ROUNDS4(abef, cdgh, msg); 282 m0 = _mm_add_epi32(m0, _mm_alignr_epi8(m3, m2, 4)); 283 m0 = _mm_sha256msg2_epu32(m0, m3); 284 m2 = _mm_sha256msg1_epu32(m2, m3); 285 286 /* Rounds 16-19 */ 287 msg = _mm_add_epi32(m0, _mm_loadu_si128((const __m128i *)&K[16])); 288 ROUNDS4(abef, cdgh, msg); 289 m1 = _mm_add_epi32(m1, _mm_alignr_epi8(m0, m3, 4)); 290 m1 = _mm_sha256msg2_epu32(m1, m0); 291 m3 = _mm_sha256msg1_epu32(m3, m0); 292 293 /* Rounds 20-23 */ 294 msg = _mm_add_epi32(m1, _mm_loadu_si128((const __m128i *)&K[20])); 295 ROUNDS4(abef, cdgh, msg); 296 m2 = _mm_add_epi32(m2, _mm_alignr_epi8(m1, m0, 4)); 297 m2 = _mm_sha256msg2_epu32(m2, m1); 298 m0 = _mm_sha256msg1_epu32(m0, m1); 299 300 /* Rounds 24-27 */ 301 msg = _mm_add_epi32(m2, _mm_loadu_si128((const __m128i *)&K[24])); 302 ROUNDS4(abef, cdgh, msg); 303 m3 = _mm_add_epi32(m3, _mm_alignr_epi8(m2, m1, 4)); 304 m3 = _mm_sha256msg2_epu32(m3, m2); 305 m1 = _mm_sha256msg1_epu32(m1, m2); 306 307 /* Rounds 28-31 */ 308 msg = _mm_add_epi32(m3, _mm_loadu_si128((const __m128i *)&K[28])); 309 ROUNDS4(abef, cdgh, msg); 310 m0 = _mm_add_epi32(m0, _mm_alignr_epi8(m3, m2, 4)); 311 m0 = _mm_sha256msg2_epu32(m0, m3); 312 m2 = _mm_sha256msg1_epu32(m2, m3); 313 314 /* Rounds 32-35 */ 315 msg = _mm_add_epi32(m0, _mm_loadu_si128((const __m128i *)&K[32])); 316 ROUNDS4(abef, cdgh, msg); 317 m1 = _mm_add_epi32(m1, _mm_alignr_epi8(m0, m3, 4)); 318 m1 = _mm_sha256msg2_epu32(m1, m0); 319 m3 = _mm_sha256msg1_epu32(m3, m0); 320 321 /* Rounds 36-39 */ 322 msg = _mm_add_epi32(m1, _mm_loadu_si128((const __m128i *)&K[36])); 323 ROUNDS4(abef, cdgh, msg); 324 m2 = _mm_add_epi32(m2, _mm_alignr_epi8(m1, m0, 4)); 325 m2 = _mm_sha256msg2_epu32(m2, m1); 326 m0 = _mm_sha256msg1_epu32(m0, m1); 327 328 /* Rounds 40-43 */ 329 msg = _mm_add_epi32(m2, _mm_loadu_si128((const __m128i *)&K[40])); 330 ROUNDS4(abef, cdgh, msg); 331 m3 = _mm_add_epi32(m3, _mm_alignr_epi8(m2, m1, 4)); 332 m3 = _mm_sha256msg2_epu32(m3, m2); 333 m1 = _mm_sha256msg1_epu32(m1, m2); 334 335 /* Rounds 44-47 */ 336 msg = _mm_add_epi32(m3, _mm_loadu_si128((const __m128i *)&K[44])); 337 ROUNDS4(abef, cdgh, msg); 338 m0 = _mm_add_epi32(m0, _mm_alignr_epi8(m3, m2, 4)); 339 m0 = _mm_sha256msg2_epu32(m0, m3); 340 m2 = _mm_sha256msg1_epu32(m2, m3); 341 342 /* Rounds 48-51 */ 343 msg = _mm_add_epi32(m0, _mm_loadu_si128((const __m128i *)&K[48])); 344 ROUNDS4(abef, cdgh, msg); 345 m1 = _mm_add_epi32(m1, _mm_alignr_epi8(m0, m3, 4)); 346 m1 = _mm_sha256msg2_epu32(m1, m0); 347 m3 = _mm_sha256msg1_epu32(m3, m0); 348 349 /* Rounds 52-55 */ 350 msg = _mm_add_epi32(m1, _mm_loadu_si128((const __m128i *)&K[52])); 351 ROUNDS4(abef, cdgh, msg); 352 m2 = _mm_add_epi32(m2, _mm_alignr_epi8(m1, m0, 4)); 353 m2 = _mm_sha256msg2_epu32(m2, m1); 354 355 /* Rounds 56-59 */ 356 msg = _mm_add_epi32(m2, _mm_loadu_si128((const __m128i *)&K[56])); 357 ROUNDS4(abef, cdgh, msg); 358 m3 = _mm_add_epi32(m3, _mm_alignr_epi8(m2, m1, 4)); 359 m3 = _mm_sha256msg2_epu32(m3, m2); 360 361 /* Rounds 60-63 */ 362 msg = _mm_add_epi32(m3, _mm_loadu_si128((const __m128i *)&K[60])); 363 ROUNDS4(abef, cdgh, msg); 364 365 /* Add original state back */ 366 abef = _mm_add_epi32(abef, abef_orig); 367 cdgh = _mm_add_epi32(cdgh, cdgh_orig); 368 369 /* Restore the state's original order and store result */ 370 tmp = _mm_shuffle_epi32(abef, 0x1b); /* f e b a */ 371 cdgh = _mm_shuffle_epi32(cdgh, 0xb1); /* d c h g */ 372 _mm_storeu_si128((__m128i *)&state[0], 373 _mm_blend_epi16(tmp, cdgh, 0xf0)); /* d c b a */ 374 _mm_storeu_si128((__m128i *)&state[4], 375 _mm_alignr_epi8(cdgh, tmp, 8)); /* h g f e */ 376 } 377 378 /* 379 * Return 1 if the CPU implements the x86 SHA extensions and the SSSE3 380 * and SSE4.1 instructions used alongside them, 0 otherwise. 381 */ 382 int sha256_native_available(void) { 383 unsigned int eax, ebx, ecx, edx; 384 385 if (!__get_cpuid(1, &eax, &ebx, &ecx, &edx)) 386 return 0; 387 if (!(ecx & bit_SSSE3) || !(ecx & bit_SSE4_1)) 388 return 0; 389 if (!__get_cpuid_count(7, 0, &eax, &ebx, &ecx, &edx)) 390 return 0; 391 return (ebx & bit_SHA) != 0; 392 } 393 394 #else 395 396 /* Stub implementations for builds without hardware support */ 397 void sha256_block_native(uint32_t *state, const uint32_t *block) { 398 (void)state; 399 (void)block; 400 /* Should never be called - use pure Haskell fallback */ 401 } 402 403 int sha256_native_available(void) { 404 return 0; 405 } 406 407 #endif