sha512_native.c (27696B)
1 #include <stdint.h> 2 #include <string.h> 3 4 /* 5 * The x86 SHA512 intrinsics arrived in GCC 14 and Clang 18; older 6 * compilers get the stubs below. 7 */ 8 #if defined(__x86_64__) && defined(__has_include) 9 #if __has_include(<sha512intrin.h>) 10 #define HAVE_X86_SHA512 11 #endif 12 #endif 13 14 #if defined(__aarch64__) 15 16 #include <arm_neon.h> 17 18 #if defined(__APPLE__) 19 #include <sys/sysctl.h> 20 #elif defined(__linux__) 21 #include <sys/auxv.h> 22 #ifndef HWCAP_SHA512 23 #define HWCAP_SHA512 (1UL << 21) 24 #endif 25 #endif 26 27 #elif defined(HAVE_X86_SHA512) 28 29 #include <cpuid.h> 30 #include <immintrin.h> 31 32 #ifndef bit_SHA512 33 #define bit_SHA512 (1 << 0) 34 #endif 35 36 #endif 37 38 #if defined(__aarch64__) || defined(HAVE_X86_SHA512) 39 40 static const uint64_t K[80] = { 41 0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL, 42 0xb5c0fbcfec4d3b2fULL, 0xe9b5dba58189dbbcULL, 43 0x3956c25bf348b538ULL, 0x59f111f1b605d019ULL, 44 0x923f82a4af194f9bULL, 0xab1c5ed5da6d8118ULL, 45 0xd807aa98a3030242ULL, 0x12835b0145706fbeULL, 46 0x243185be4ee4b28cULL, 0x550c7dc3d5ffb4e2ULL, 47 0x72be5d74f27b896fULL, 0x80deb1fe3b1696b1ULL, 48 0x9bdc06a725c71235ULL, 0xc19bf174cf692694ULL, 49 0xe49b69c19ef14ad2ULL, 0xefbe4786384f25e3ULL, 50 0x0fc19dc68b8cd5b5ULL, 0x240ca1cc77ac9c65ULL, 51 0x2de92c6f592b0275ULL, 0x4a7484aa6ea6e483ULL, 52 0x5cb0a9dcbd41fbd4ULL, 0x76f988da831153b5ULL, 53 0x983e5152ee66dfabULL, 0xa831c66d2db43210ULL, 54 0xb00327c898fb213fULL, 0xbf597fc7beef0ee4ULL, 55 0xc6e00bf33da88fc2ULL, 0xd5a79147930aa725ULL, 56 0x06ca6351e003826fULL, 0x142929670a0e6e70ULL, 57 0x27b70a8546d22ffcULL, 0x2e1b21385c26c926ULL, 58 0x4d2c6dfc5ac42aedULL, 0x53380d139d95b3dfULL, 59 0x650a73548baf63deULL, 0x766a0abb3c77b2a8ULL, 60 0x81c2c92e47edaee6ULL, 0x92722c851482353bULL, 61 0xa2bfe8a14cf10364ULL, 0xa81a664bbc423001ULL, 62 0xc24b8b70d0f89791ULL, 0xc76c51a30654be30ULL, 63 0xd192e819d6ef5218ULL, 0xd69906245565a910ULL, 64 0xf40e35855771202aULL, 0x106aa07032bbd1b8ULL, 65 0x19a4c116b8d2d0c8ULL, 0x1e376c085141ab53ULL, 66 0x2748774cdf8eeb99ULL, 0x34b0bcb5e19b48a8ULL, 67 0x391c0cb3c5c95a63ULL, 0x4ed8aa4ae3418acbULL, 68 0x5b9cca4f7763e373ULL, 0x682e6ff3d6b2b8a3ULL, 69 0x748f82ee5defb2fcULL, 0x78a5636f43172f60ULL, 70 0x84c87814a1f0ab72ULL, 0x8cc702081a6439ecULL, 71 0x90befffa23631e28ULL, 0xa4506cebde82bde9ULL, 72 0xbef9a3f7b2c67915ULL, 0xc67178f2e372532bULL, 73 0xca273eceea26619cULL, 0xd186b8c721c0c207ULL, 74 0xeada7dd6cde0eb1eULL, 0xf57d4f7fee6ed178ULL, 75 0x06f067aa72176fbaULL, 0x0a637dc5a2c898a6ULL, 76 0x113f9804bef90daeULL, 0x1b710b35131c471bULL, 77 0x28db77f523047d84ULL, 0x32caab7b40c72493ULL, 78 0x3c9ebe0a15c9bebcULL, 0x431d67c49c100d4cULL, 79 0x4cc5d4becb3e42b6ULL, 0x597f299cfc657e2aULL, 80 0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL 81 }; 82 83 #endif 84 85 #if defined(__aarch64__) 86 87 /* 88 * Process one 128-byte block using ARM SHA512 crypto instructions. 89 * 90 * state: pointer to 8 uint64_t words (a,b,c,d,e,f,g,h) 91 * block: pointer to 16 uint64_t words (already native endian) 92 * 93 * The state is updated in place. 94 * 95 * Compiled for the SHA512 extension only; callers must first check 96 * 'sha512_native_available'. 97 */ 98 __attribute__((target("+sha3"))) 99 void sha512_block_native(uint64_t *state, const uint64_t *block) { 100 /* Load current hash state */ 101 uint64x2_t ab = vld1q_u64(&state[0]); 102 uint64x2_t cd = vld1q_u64(&state[2]); 103 uint64x2_t ef = vld1q_u64(&state[4]); 104 uint64x2_t gh = vld1q_u64(&state[6]); 105 106 /* Save original for final addition */ 107 uint64x2_t ab_orig = ab; 108 uint64x2_t cd_orig = cd; 109 uint64x2_t ef_orig = ef; 110 uint64x2_t gh_orig = gh; 111 112 /* Load message (already native endian) */ 113 uint64x2_t m0 = vld1q_u64(&block[0]); 114 uint64x2_t m1 = vld1q_u64(&block[2]); 115 uint64x2_t m2 = vld1q_u64(&block[4]); 116 uint64x2_t m3 = vld1q_u64(&block[6]); 117 uint64x2_t m4 = vld1q_u64(&block[8]); 118 uint64x2_t m5 = vld1q_u64(&block[10]); 119 uint64x2_t m6 = vld1q_u64(&block[12]); 120 uint64x2_t m7 = vld1q_u64(&block[14]); 121 122 uint64x2_t tmp; 123 124 /* Rounds 0-1 */ 125 tmp = vaddq_u64(m0, vld1q_u64(&K[0])); 126 tmp = vextq_u64(tmp, tmp, 1); 127 tmp = vaddq_u64(gh, tmp); 128 tmp = vsha512hq_u64(tmp, vextq_u64(ef, gh, 1), vextq_u64(cd, ef, 1)); 129 gh = vsha512h2q_u64(tmp, cd, ab); 130 cd = vaddq_u64(cd, tmp); 131 m0 = vsha512su1q_u64(vsha512su0q_u64(m0, m1), m7, vextq_u64(m4, m5, 1)); 132 133 /* Rounds 2-3 */ 134 tmp = vaddq_u64(m1, vld1q_u64(&K[2])); 135 tmp = vextq_u64(tmp, tmp, 1); 136 tmp = vaddq_u64(ef, tmp); 137 tmp = vsha512hq_u64(tmp, vextq_u64(cd, ef, 1), vextq_u64(ab, cd, 1)); 138 ef = vsha512h2q_u64(tmp, ab, gh); 139 ab = vaddq_u64(ab, tmp); 140 m1 = vsha512su1q_u64(vsha512su0q_u64(m1, m2), m0, vextq_u64(m5, m6, 1)); 141 142 /* Rounds 4-5 */ 143 tmp = vaddq_u64(m2, vld1q_u64(&K[4])); 144 tmp = vextq_u64(tmp, tmp, 1); 145 tmp = vaddq_u64(cd, tmp); 146 tmp = vsha512hq_u64(tmp, vextq_u64(ab, cd, 1), vextq_u64(gh, ab, 1)); 147 cd = vsha512h2q_u64(tmp, gh, ef); 148 gh = vaddq_u64(gh, tmp); 149 m2 = vsha512su1q_u64(vsha512su0q_u64(m2, m3), m1, vextq_u64(m6, m7, 1)); 150 151 /* Rounds 6-7 */ 152 tmp = vaddq_u64(m3, vld1q_u64(&K[6])); 153 tmp = vextq_u64(tmp, tmp, 1); 154 tmp = vaddq_u64(ab, tmp); 155 tmp = vsha512hq_u64(tmp, vextq_u64(gh, ab, 1), vextq_u64(ef, gh, 1)); 156 ab = vsha512h2q_u64(tmp, ef, cd); 157 ef = vaddq_u64(ef, tmp); 158 m3 = vsha512su1q_u64(vsha512su0q_u64(m3, m4), m2, vextq_u64(m7, m0, 1)); 159 160 /* Rounds 8-9 */ 161 tmp = vaddq_u64(m4, vld1q_u64(&K[8])); 162 tmp = vextq_u64(tmp, tmp, 1); 163 tmp = vaddq_u64(gh, tmp); 164 tmp = vsha512hq_u64(tmp, vextq_u64(ef, gh, 1), vextq_u64(cd, ef, 1)); 165 gh = vsha512h2q_u64(tmp, cd, ab); 166 cd = vaddq_u64(cd, tmp); 167 m4 = vsha512su1q_u64(vsha512su0q_u64(m4, m5), m3, vextq_u64(m0, m1, 1)); 168 169 /* Rounds 10-11 */ 170 tmp = vaddq_u64(m5, vld1q_u64(&K[10])); 171 tmp = vextq_u64(tmp, tmp, 1); 172 tmp = vaddq_u64(ef, tmp); 173 tmp = vsha512hq_u64(tmp, vextq_u64(cd, ef, 1), vextq_u64(ab, cd, 1)); 174 ef = vsha512h2q_u64(tmp, ab, gh); 175 ab = vaddq_u64(ab, tmp); 176 m5 = vsha512su1q_u64(vsha512su0q_u64(m5, m6), m4, vextq_u64(m1, m2, 1)); 177 178 /* Rounds 12-13 */ 179 tmp = vaddq_u64(m6, vld1q_u64(&K[12])); 180 tmp = vextq_u64(tmp, tmp, 1); 181 tmp = vaddq_u64(cd, tmp); 182 tmp = vsha512hq_u64(tmp, vextq_u64(ab, cd, 1), vextq_u64(gh, ab, 1)); 183 cd = vsha512h2q_u64(tmp, gh, ef); 184 gh = vaddq_u64(gh, tmp); 185 m6 = vsha512su1q_u64(vsha512su0q_u64(m6, m7), m5, vextq_u64(m2, m3, 1)); 186 187 /* Rounds 14-15 */ 188 tmp = vaddq_u64(m7, vld1q_u64(&K[14])); 189 tmp = vextq_u64(tmp, tmp, 1); 190 tmp = vaddq_u64(ab, tmp); 191 tmp = vsha512hq_u64(tmp, vextq_u64(gh, ab, 1), vextq_u64(ef, gh, 1)); 192 ab = vsha512h2q_u64(tmp, ef, cd); 193 ef = vaddq_u64(ef, tmp); 194 m7 = vsha512su1q_u64(vsha512su0q_u64(m7, m0), m6, vextq_u64(m3, m4, 1)); 195 196 /* Rounds 16-17 */ 197 tmp = vaddq_u64(m0, vld1q_u64(&K[16])); 198 tmp = vextq_u64(tmp, tmp, 1); 199 tmp = vaddq_u64(gh, tmp); 200 tmp = vsha512hq_u64(tmp, vextq_u64(ef, gh, 1), vextq_u64(cd, ef, 1)); 201 gh = vsha512h2q_u64(tmp, cd, ab); 202 cd = vaddq_u64(cd, tmp); 203 m0 = vsha512su1q_u64(vsha512su0q_u64(m0, m1), m7, vextq_u64(m4, m5, 1)); 204 205 /* Rounds 18-19 */ 206 tmp = vaddq_u64(m1, vld1q_u64(&K[18])); 207 tmp = vextq_u64(tmp, tmp, 1); 208 tmp = vaddq_u64(ef, tmp); 209 tmp = vsha512hq_u64(tmp, vextq_u64(cd, ef, 1), vextq_u64(ab, cd, 1)); 210 ef = vsha512h2q_u64(tmp, ab, gh); 211 ab = vaddq_u64(ab, tmp); 212 m1 = vsha512su1q_u64(vsha512su0q_u64(m1, m2), m0, vextq_u64(m5, m6, 1)); 213 214 /* Rounds 20-21 */ 215 tmp = vaddq_u64(m2, vld1q_u64(&K[20])); 216 tmp = vextq_u64(tmp, tmp, 1); 217 tmp = vaddq_u64(cd, tmp); 218 tmp = vsha512hq_u64(tmp, vextq_u64(ab, cd, 1), vextq_u64(gh, ab, 1)); 219 cd = vsha512h2q_u64(tmp, gh, ef); 220 gh = vaddq_u64(gh, tmp); 221 m2 = vsha512su1q_u64(vsha512su0q_u64(m2, m3), m1, vextq_u64(m6, m7, 1)); 222 223 /* Rounds 22-23 */ 224 tmp = vaddq_u64(m3, vld1q_u64(&K[22])); 225 tmp = vextq_u64(tmp, tmp, 1); 226 tmp = vaddq_u64(ab, tmp); 227 tmp = vsha512hq_u64(tmp, vextq_u64(gh, ab, 1), vextq_u64(ef, gh, 1)); 228 ab = vsha512h2q_u64(tmp, ef, cd); 229 ef = vaddq_u64(ef, tmp); 230 m3 = vsha512su1q_u64(vsha512su0q_u64(m3, m4), m2, vextq_u64(m7, m0, 1)); 231 232 /* Rounds 24-25 */ 233 tmp = vaddq_u64(m4, vld1q_u64(&K[24])); 234 tmp = vextq_u64(tmp, tmp, 1); 235 tmp = vaddq_u64(gh, tmp); 236 tmp = vsha512hq_u64(tmp, vextq_u64(ef, gh, 1), vextq_u64(cd, ef, 1)); 237 gh = vsha512h2q_u64(tmp, cd, ab); 238 cd = vaddq_u64(cd, tmp); 239 m4 = vsha512su1q_u64(vsha512su0q_u64(m4, m5), m3, vextq_u64(m0, m1, 1)); 240 241 /* Rounds 26-27 */ 242 tmp = vaddq_u64(m5, vld1q_u64(&K[26])); 243 tmp = vextq_u64(tmp, tmp, 1); 244 tmp = vaddq_u64(ef, tmp); 245 tmp = vsha512hq_u64(tmp, vextq_u64(cd, ef, 1), vextq_u64(ab, cd, 1)); 246 ef = vsha512h2q_u64(tmp, ab, gh); 247 ab = vaddq_u64(ab, tmp); 248 m5 = vsha512su1q_u64(vsha512su0q_u64(m5, m6), m4, vextq_u64(m1, m2, 1)); 249 250 /* Rounds 28-29 */ 251 tmp = vaddq_u64(m6, vld1q_u64(&K[28])); 252 tmp = vextq_u64(tmp, tmp, 1); 253 tmp = vaddq_u64(cd, tmp); 254 tmp = vsha512hq_u64(tmp, vextq_u64(ab, cd, 1), vextq_u64(gh, ab, 1)); 255 cd = vsha512h2q_u64(tmp, gh, ef); 256 gh = vaddq_u64(gh, tmp); 257 m6 = vsha512su1q_u64(vsha512su0q_u64(m6, m7), m5, vextq_u64(m2, m3, 1)); 258 259 /* Rounds 30-31 */ 260 tmp = vaddq_u64(m7, vld1q_u64(&K[30])); 261 tmp = vextq_u64(tmp, tmp, 1); 262 tmp = vaddq_u64(ab, tmp); 263 tmp = vsha512hq_u64(tmp, vextq_u64(gh, ab, 1), vextq_u64(ef, gh, 1)); 264 ab = vsha512h2q_u64(tmp, ef, cd); 265 ef = vaddq_u64(ef, tmp); 266 m7 = vsha512su1q_u64(vsha512su0q_u64(m7, m0), m6, vextq_u64(m3, m4, 1)); 267 268 /* Rounds 32-33 */ 269 tmp = vaddq_u64(m0, vld1q_u64(&K[32])); 270 tmp = vextq_u64(tmp, tmp, 1); 271 tmp = vaddq_u64(gh, tmp); 272 tmp = vsha512hq_u64(tmp, vextq_u64(ef, gh, 1), vextq_u64(cd, ef, 1)); 273 gh = vsha512h2q_u64(tmp, cd, ab); 274 cd = vaddq_u64(cd, tmp); 275 m0 = vsha512su1q_u64(vsha512su0q_u64(m0, m1), m7, vextq_u64(m4, m5, 1)); 276 277 /* Rounds 34-35 */ 278 tmp = vaddq_u64(m1, vld1q_u64(&K[34])); 279 tmp = vextq_u64(tmp, tmp, 1); 280 tmp = vaddq_u64(ef, tmp); 281 tmp = vsha512hq_u64(tmp, vextq_u64(cd, ef, 1), vextq_u64(ab, cd, 1)); 282 ef = vsha512h2q_u64(tmp, ab, gh); 283 ab = vaddq_u64(ab, tmp); 284 m1 = vsha512su1q_u64(vsha512su0q_u64(m1, m2), m0, vextq_u64(m5, m6, 1)); 285 286 /* Rounds 36-37 */ 287 tmp = vaddq_u64(m2, vld1q_u64(&K[36])); 288 tmp = vextq_u64(tmp, tmp, 1); 289 tmp = vaddq_u64(cd, tmp); 290 tmp = vsha512hq_u64(tmp, vextq_u64(ab, cd, 1), vextq_u64(gh, ab, 1)); 291 cd = vsha512h2q_u64(tmp, gh, ef); 292 gh = vaddq_u64(gh, tmp); 293 m2 = vsha512su1q_u64(vsha512su0q_u64(m2, m3), m1, vextq_u64(m6, m7, 1)); 294 295 /* Rounds 38-39 */ 296 tmp = vaddq_u64(m3, vld1q_u64(&K[38])); 297 tmp = vextq_u64(tmp, tmp, 1); 298 tmp = vaddq_u64(ab, tmp); 299 tmp = vsha512hq_u64(tmp, vextq_u64(gh, ab, 1), vextq_u64(ef, gh, 1)); 300 ab = vsha512h2q_u64(tmp, ef, cd); 301 ef = vaddq_u64(ef, tmp); 302 m3 = vsha512su1q_u64(vsha512su0q_u64(m3, m4), m2, vextq_u64(m7, m0, 1)); 303 304 /* Rounds 40-41 */ 305 tmp = vaddq_u64(m4, vld1q_u64(&K[40])); 306 tmp = vextq_u64(tmp, tmp, 1); 307 tmp = vaddq_u64(gh, tmp); 308 tmp = vsha512hq_u64(tmp, vextq_u64(ef, gh, 1), vextq_u64(cd, ef, 1)); 309 gh = vsha512h2q_u64(tmp, cd, ab); 310 cd = vaddq_u64(cd, tmp); 311 m4 = vsha512su1q_u64(vsha512su0q_u64(m4, m5), m3, vextq_u64(m0, m1, 1)); 312 313 /* Rounds 42-43 */ 314 tmp = vaddq_u64(m5, vld1q_u64(&K[42])); 315 tmp = vextq_u64(tmp, tmp, 1); 316 tmp = vaddq_u64(ef, tmp); 317 tmp = vsha512hq_u64(tmp, vextq_u64(cd, ef, 1), vextq_u64(ab, cd, 1)); 318 ef = vsha512h2q_u64(tmp, ab, gh); 319 ab = vaddq_u64(ab, tmp); 320 m5 = vsha512su1q_u64(vsha512su0q_u64(m5, m6), m4, vextq_u64(m1, m2, 1)); 321 322 /* Rounds 44-45 */ 323 tmp = vaddq_u64(m6, vld1q_u64(&K[44])); 324 tmp = vextq_u64(tmp, tmp, 1); 325 tmp = vaddq_u64(cd, tmp); 326 tmp = vsha512hq_u64(tmp, vextq_u64(ab, cd, 1), vextq_u64(gh, ab, 1)); 327 cd = vsha512h2q_u64(tmp, gh, ef); 328 gh = vaddq_u64(gh, tmp); 329 m6 = vsha512su1q_u64(vsha512su0q_u64(m6, m7), m5, vextq_u64(m2, m3, 1)); 330 331 /* Rounds 46-47 */ 332 tmp = vaddq_u64(m7, vld1q_u64(&K[46])); 333 tmp = vextq_u64(tmp, tmp, 1); 334 tmp = vaddq_u64(ab, tmp); 335 tmp = vsha512hq_u64(tmp, vextq_u64(gh, ab, 1), vextq_u64(ef, gh, 1)); 336 ab = vsha512h2q_u64(tmp, ef, cd); 337 ef = vaddq_u64(ef, tmp); 338 m7 = vsha512su1q_u64(vsha512su0q_u64(m7, m0), m6, vextq_u64(m3, m4, 1)); 339 340 /* Rounds 48-49 */ 341 tmp = vaddq_u64(m0, vld1q_u64(&K[48])); 342 tmp = vextq_u64(tmp, tmp, 1); 343 tmp = vaddq_u64(gh, tmp); 344 tmp = vsha512hq_u64(tmp, vextq_u64(ef, gh, 1), vextq_u64(cd, ef, 1)); 345 gh = vsha512h2q_u64(tmp, cd, ab); 346 cd = vaddq_u64(cd, tmp); 347 m0 = vsha512su1q_u64(vsha512su0q_u64(m0, m1), m7, vextq_u64(m4, m5, 1)); 348 349 /* Rounds 50-51 */ 350 tmp = vaddq_u64(m1, vld1q_u64(&K[50])); 351 tmp = vextq_u64(tmp, tmp, 1); 352 tmp = vaddq_u64(ef, tmp); 353 tmp = vsha512hq_u64(tmp, vextq_u64(cd, ef, 1), vextq_u64(ab, cd, 1)); 354 ef = vsha512h2q_u64(tmp, ab, gh); 355 ab = vaddq_u64(ab, tmp); 356 m1 = vsha512su1q_u64(vsha512su0q_u64(m1, m2), m0, vextq_u64(m5, m6, 1)); 357 358 /* Rounds 52-53 */ 359 tmp = vaddq_u64(m2, vld1q_u64(&K[52])); 360 tmp = vextq_u64(tmp, tmp, 1); 361 tmp = vaddq_u64(cd, tmp); 362 tmp = vsha512hq_u64(tmp, vextq_u64(ab, cd, 1), vextq_u64(gh, ab, 1)); 363 cd = vsha512h2q_u64(tmp, gh, ef); 364 gh = vaddq_u64(gh, tmp); 365 m2 = vsha512su1q_u64(vsha512su0q_u64(m2, m3), m1, vextq_u64(m6, m7, 1)); 366 367 /* Rounds 54-55 */ 368 tmp = vaddq_u64(m3, vld1q_u64(&K[54])); 369 tmp = vextq_u64(tmp, tmp, 1); 370 tmp = vaddq_u64(ab, tmp); 371 tmp = vsha512hq_u64(tmp, vextq_u64(gh, ab, 1), vextq_u64(ef, gh, 1)); 372 ab = vsha512h2q_u64(tmp, ef, cd); 373 ef = vaddq_u64(ef, tmp); 374 m3 = vsha512su1q_u64(vsha512su0q_u64(m3, m4), m2, vextq_u64(m7, m0, 1)); 375 376 /* Rounds 56-57 */ 377 tmp = vaddq_u64(m4, vld1q_u64(&K[56])); 378 tmp = vextq_u64(tmp, tmp, 1); 379 tmp = vaddq_u64(gh, tmp); 380 tmp = vsha512hq_u64(tmp, vextq_u64(ef, gh, 1), vextq_u64(cd, ef, 1)); 381 gh = vsha512h2q_u64(tmp, cd, ab); 382 cd = vaddq_u64(cd, tmp); 383 m4 = vsha512su1q_u64(vsha512su0q_u64(m4, m5), m3, vextq_u64(m0, m1, 1)); 384 385 /* Rounds 58-59 */ 386 tmp = vaddq_u64(m5, vld1q_u64(&K[58])); 387 tmp = vextq_u64(tmp, tmp, 1); 388 tmp = vaddq_u64(ef, tmp); 389 tmp = vsha512hq_u64(tmp, vextq_u64(cd, ef, 1), vextq_u64(ab, cd, 1)); 390 ef = vsha512h2q_u64(tmp, ab, gh); 391 ab = vaddq_u64(ab, tmp); 392 m5 = vsha512su1q_u64(vsha512su0q_u64(m5, m6), m4, vextq_u64(m1, m2, 1)); 393 394 /* Rounds 60-61 */ 395 tmp = vaddq_u64(m6, vld1q_u64(&K[60])); 396 tmp = vextq_u64(tmp, tmp, 1); 397 tmp = vaddq_u64(cd, tmp); 398 tmp = vsha512hq_u64(tmp, vextq_u64(ab, cd, 1), vextq_u64(gh, ab, 1)); 399 cd = vsha512h2q_u64(tmp, gh, ef); 400 gh = vaddq_u64(gh, tmp); 401 m6 = vsha512su1q_u64(vsha512su0q_u64(m6, m7), m5, vextq_u64(m2, m3, 1)); 402 403 /* Rounds 62-63 */ 404 tmp = vaddq_u64(m7, vld1q_u64(&K[62])); 405 tmp = vextq_u64(tmp, tmp, 1); 406 tmp = vaddq_u64(ab, tmp); 407 tmp = vsha512hq_u64(tmp, vextq_u64(gh, ab, 1), vextq_u64(ef, gh, 1)); 408 ab = vsha512h2q_u64(tmp, ef, cd); 409 ef = vaddq_u64(ef, tmp); 410 m7 = vsha512su1q_u64(vsha512su0q_u64(m7, m0), m6, vextq_u64(m3, m4, 1)); 411 412 /* Rounds 64-65 */ 413 tmp = vaddq_u64(m0, vld1q_u64(&K[64])); 414 tmp = vextq_u64(tmp, tmp, 1); 415 tmp = vaddq_u64(gh, tmp); 416 tmp = vsha512hq_u64(tmp, vextq_u64(ef, gh, 1), vextq_u64(cd, ef, 1)); 417 gh = vsha512h2q_u64(tmp, cd, ab); 418 cd = vaddq_u64(cd, tmp); 419 420 /* Rounds 66-67 */ 421 tmp = vaddq_u64(m1, vld1q_u64(&K[66])); 422 tmp = vextq_u64(tmp, tmp, 1); 423 tmp = vaddq_u64(ef, tmp); 424 tmp = vsha512hq_u64(tmp, vextq_u64(cd, ef, 1), vextq_u64(ab, cd, 1)); 425 ef = vsha512h2q_u64(tmp, ab, gh); 426 ab = vaddq_u64(ab, tmp); 427 428 /* Rounds 68-69 */ 429 tmp = vaddq_u64(m2, vld1q_u64(&K[68])); 430 tmp = vextq_u64(tmp, tmp, 1); 431 tmp = vaddq_u64(cd, tmp); 432 tmp = vsha512hq_u64(tmp, vextq_u64(ab, cd, 1), vextq_u64(gh, ab, 1)); 433 cd = vsha512h2q_u64(tmp, gh, ef); 434 gh = vaddq_u64(gh, tmp); 435 436 /* Rounds 70-71 */ 437 tmp = vaddq_u64(m3, vld1q_u64(&K[70])); 438 tmp = vextq_u64(tmp, tmp, 1); 439 tmp = vaddq_u64(ab, tmp); 440 tmp = vsha512hq_u64(tmp, vextq_u64(gh, ab, 1), vextq_u64(ef, gh, 1)); 441 ab = vsha512h2q_u64(tmp, ef, cd); 442 ef = vaddq_u64(ef, tmp); 443 444 /* Rounds 72-73 */ 445 tmp = vaddq_u64(m4, vld1q_u64(&K[72])); 446 tmp = vextq_u64(tmp, tmp, 1); 447 tmp = vaddq_u64(gh, tmp); 448 tmp = vsha512hq_u64(tmp, vextq_u64(ef, gh, 1), vextq_u64(cd, ef, 1)); 449 gh = vsha512h2q_u64(tmp, cd, ab); 450 cd = vaddq_u64(cd, tmp); 451 452 /* Rounds 74-75 */ 453 tmp = vaddq_u64(m5, vld1q_u64(&K[74])); 454 tmp = vextq_u64(tmp, tmp, 1); 455 tmp = vaddq_u64(ef, tmp); 456 tmp = vsha512hq_u64(tmp, vextq_u64(cd, ef, 1), vextq_u64(ab, cd, 1)); 457 ef = vsha512h2q_u64(tmp, ab, gh); 458 ab = vaddq_u64(ab, tmp); 459 460 /* Rounds 76-77 */ 461 tmp = vaddq_u64(m6, vld1q_u64(&K[76])); 462 tmp = vextq_u64(tmp, tmp, 1); 463 tmp = vaddq_u64(cd, tmp); 464 tmp = vsha512hq_u64(tmp, vextq_u64(ab, cd, 1), vextq_u64(gh, ab, 1)); 465 cd = vsha512h2q_u64(tmp, gh, ef); 466 gh = vaddq_u64(gh, tmp); 467 468 /* Rounds 78-79 */ 469 tmp = vaddq_u64(m7, vld1q_u64(&K[78])); 470 tmp = vextq_u64(tmp, tmp, 1); 471 tmp = vaddq_u64(ab, tmp); 472 tmp = vsha512hq_u64(tmp, vextq_u64(gh, ab, 1), vextq_u64(ef, gh, 1)); 473 ab = vsha512h2q_u64(tmp, ef, cd); 474 ef = vaddq_u64(ef, tmp); 475 476 /* Add original state back */ 477 ab = vaddq_u64(ab, ab_orig); 478 cd = vaddq_u64(cd, cd_orig); 479 ef = vaddq_u64(ef, ef_orig); 480 gh = vaddq_u64(gh, gh_orig); 481 482 /* Store result */ 483 vst1q_u64(&state[0], ab); 484 vst1q_u64(&state[2], cd); 485 vst1q_u64(&state[4], ef); 486 vst1q_u64(&state[6], gh); 487 } 488 489 #if defined(__APPLE__) 490 static int sysctl_flag(const char *name) { 491 int val = 0; 492 size_t len = sizeof(val); 493 if (sysctlbyname(name, &val, &len, NULL, 0) != 0) return 0; 494 return val != 0; 495 } 496 #endif 497 498 /* 499 * Return 1 if the CPU implements the ARM SHA512 extension, 0 500 * otherwise. We only know how to ask Darwin and Linux, and report 0 501 * (selecting the pure Haskell fallback) on any other OS. 502 */ 503 int sha512_native_available(void) { 504 #if defined(__APPLE__) 505 return sysctl_flag("hw.optional.arm.FEAT_SHA512") 506 || sysctl_flag("hw.optional.armv8_2_sha512"); 507 #elif defined(__linux__) 508 return (getauxval(AT_HWCAP) & HWCAP_SHA512) != 0; 509 #else 510 return 0; 511 #endif 512 } 513 514 #elif defined(HAVE_X86_SHA512) 515 516 /* 517 * Four rounds. 'sha512rnds2' does two: it takes the (c,d,g,h) and 518 * (a,b,e,f) halves of the state, and two message-plus-constant words 519 * in its third argument, and returns the new (a,b,e,f). The old 520 * (a,b,e,f) is then the new (c,d,g,h), so a second call with the 521 * halves swapped, on the upper two words, does the next two. 522 */ 523 #define ROUNDS4(abef, cdgh, msg) \ 524 do { \ 525 cdgh = _mm256_sha512rnds2_epi64(cdgh, abef, \ 526 _mm256_castsi256_si128(msg)); \ 527 abef = _mm256_sha512rnds2_epi64(abef, cdgh, \ 528 _mm256_extracti128_si256(msg, 1)); \ 529 } while (0) 530 531 /* 532 * The four words following prev[0], i.e. (prev[1], prev[2], prev[3], 533 * cur[0]) from lowest lane to highest. 534 */ 535 #define ALIGNR1(cur, prev) \ 536 _mm256_permute4x64_epi64(_mm256_blend_epi32(prev, cur, 0x03), 0x39) 537 538 /* 539 * Process one 128-byte block using x86 SHA512 instructions. 540 * 541 * state: pointer to 8 uint64_t words (a,b,c,d,e,f,g,h) 542 * block: pointer to 16 uint64_t words (already native endian) 543 * 544 * The state is updated in place. 545 * 546 * Compiled for the SHA512 and AVX2 extensions only; callers must first 547 * check 'sha512_native_available'. 548 * 549 * Lane orders in comments are written from highest lane to lowest. 550 */ 551 __attribute__((target("avx2,sha512"))) 552 void sha512_block_native(uint64_t *state, const uint64_t *block) { 553 __m256i abcd, efgh, abef, cdgh, abef_orig, cdgh_orig; 554 __m256i m0, m1, m2, m3, msg; 555 556 /* Load current hash state, rearranging it for the SHA instructions */ 557 abcd = _mm256_loadu_si256((const __m256i *)&state[0]); /* d c b a */ 558 efgh = _mm256_loadu_si256((const __m256i *)&state[4]); /* h g f e */ 559 abef = _mm256_permute4x64_epi64( 560 _mm256_permute2x128_si256(abcd, efgh, 0x20), 0x1b); 561 /* a b e f */ 562 cdgh = _mm256_permute4x64_epi64( 563 _mm256_permute2x128_si256(abcd, efgh, 0x31), 0x1b); 564 /* c d g h */ 565 566 /* Save original for final addition */ 567 abef_orig = abef; 568 cdgh_orig = cdgh; 569 570 /* Load message (already native endian) */ 571 m0 = _mm256_loadu_si256((const __m256i *)&block[0]); 572 m1 = _mm256_loadu_si256((const __m256i *)&block[4]); 573 m2 = _mm256_loadu_si256((const __m256i *)&block[8]); 574 m3 = _mm256_loadu_si256((const __m256i *)&block[12]); 575 576 /* Rounds 0-3 */ 577 msg = _mm256_add_epi64(m0, _mm256_loadu_si256((const __m256i *)&K[0])); 578 ROUNDS4(abef, cdgh, msg); 579 580 /* Rounds 4-7 */ 581 msg = _mm256_add_epi64(m1, _mm256_loadu_si256((const __m256i *)&K[4])); 582 ROUNDS4(abef, cdgh, msg); 583 m0 = _mm256_sha512msg1_epi64(m0, _mm256_castsi256_si128(m1)); 584 585 /* Rounds 8-11 */ 586 msg = _mm256_add_epi64(m2, _mm256_loadu_si256((const __m256i *)&K[8])); 587 ROUNDS4(abef, cdgh, msg); 588 m1 = _mm256_sha512msg1_epi64(m1, _mm256_castsi256_si128(m2)); 589 590 /* Rounds 12-15 */ 591 msg = _mm256_add_epi64(m3, _mm256_loadu_si256((const __m256i *)&K[12])); 592 ROUNDS4(abef, cdgh, msg); 593 m0 = _mm256_add_epi64(m0, ALIGNR1(m3, m2)); 594 m0 = _mm256_sha512msg2_epi64(m0, m3); 595 m2 = _mm256_sha512msg1_epi64(m2, _mm256_castsi256_si128(m3)); 596 597 /* Rounds 16-19 */ 598 msg = _mm256_add_epi64(m0, _mm256_loadu_si256((const __m256i *)&K[16])); 599 ROUNDS4(abef, cdgh, msg); 600 m1 = _mm256_add_epi64(m1, ALIGNR1(m0, m3)); 601 m1 = _mm256_sha512msg2_epi64(m1, m0); 602 m3 = _mm256_sha512msg1_epi64(m3, _mm256_castsi256_si128(m0)); 603 604 /* Rounds 20-23 */ 605 msg = _mm256_add_epi64(m1, _mm256_loadu_si256((const __m256i *)&K[20])); 606 ROUNDS4(abef, cdgh, msg); 607 m2 = _mm256_add_epi64(m2, ALIGNR1(m1, m0)); 608 m2 = _mm256_sha512msg2_epi64(m2, m1); 609 m0 = _mm256_sha512msg1_epi64(m0, _mm256_castsi256_si128(m1)); 610 611 /* Rounds 24-27 */ 612 msg = _mm256_add_epi64(m2, _mm256_loadu_si256((const __m256i *)&K[24])); 613 ROUNDS4(abef, cdgh, msg); 614 m3 = _mm256_add_epi64(m3, ALIGNR1(m2, m1)); 615 m3 = _mm256_sha512msg2_epi64(m3, m2); 616 m1 = _mm256_sha512msg1_epi64(m1, _mm256_castsi256_si128(m2)); 617 618 /* Rounds 28-31 */ 619 msg = _mm256_add_epi64(m3, _mm256_loadu_si256((const __m256i *)&K[28])); 620 ROUNDS4(abef, cdgh, msg); 621 m0 = _mm256_add_epi64(m0, ALIGNR1(m3, m2)); 622 m0 = _mm256_sha512msg2_epi64(m0, m3); 623 m2 = _mm256_sha512msg1_epi64(m2, _mm256_castsi256_si128(m3)); 624 625 /* Rounds 32-35 */ 626 msg = _mm256_add_epi64(m0, _mm256_loadu_si256((const __m256i *)&K[32])); 627 ROUNDS4(abef, cdgh, msg); 628 m1 = _mm256_add_epi64(m1, ALIGNR1(m0, m3)); 629 m1 = _mm256_sha512msg2_epi64(m1, m0); 630 m3 = _mm256_sha512msg1_epi64(m3, _mm256_castsi256_si128(m0)); 631 632 /* Rounds 36-39 */ 633 msg = _mm256_add_epi64(m1, _mm256_loadu_si256((const __m256i *)&K[36])); 634 ROUNDS4(abef, cdgh, msg); 635 m2 = _mm256_add_epi64(m2, ALIGNR1(m1, m0)); 636 m2 = _mm256_sha512msg2_epi64(m2, m1); 637 m0 = _mm256_sha512msg1_epi64(m0, _mm256_castsi256_si128(m1)); 638 639 /* Rounds 40-43 */ 640 msg = _mm256_add_epi64(m2, _mm256_loadu_si256((const __m256i *)&K[40])); 641 ROUNDS4(abef, cdgh, msg); 642 m3 = _mm256_add_epi64(m3, ALIGNR1(m2, m1)); 643 m3 = _mm256_sha512msg2_epi64(m3, m2); 644 m1 = _mm256_sha512msg1_epi64(m1, _mm256_castsi256_si128(m2)); 645 646 /* Rounds 44-47 */ 647 msg = _mm256_add_epi64(m3, _mm256_loadu_si256((const __m256i *)&K[44])); 648 ROUNDS4(abef, cdgh, msg); 649 m0 = _mm256_add_epi64(m0, ALIGNR1(m3, m2)); 650 m0 = _mm256_sha512msg2_epi64(m0, m3); 651 m2 = _mm256_sha512msg1_epi64(m2, _mm256_castsi256_si128(m3)); 652 653 /* Rounds 48-51 */ 654 msg = _mm256_add_epi64(m0, _mm256_loadu_si256((const __m256i *)&K[48])); 655 ROUNDS4(abef, cdgh, msg); 656 m1 = _mm256_add_epi64(m1, ALIGNR1(m0, m3)); 657 m1 = _mm256_sha512msg2_epi64(m1, m0); 658 m3 = _mm256_sha512msg1_epi64(m3, _mm256_castsi256_si128(m0)); 659 660 /* Rounds 52-55 */ 661 msg = _mm256_add_epi64(m1, _mm256_loadu_si256((const __m256i *)&K[52])); 662 ROUNDS4(abef, cdgh, msg); 663 m2 = _mm256_add_epi64(m2, ALIGNR1(m1, m0)); 664 m2 = _mm256_sha512msg2_epi64(m2, m1); 665 m0 = _mm256_sha512msg1_epi64(m0, _mm256_castsi256_si128(m1)); 666 667 /* Rounds 56-59 */ 668 msg = _mm256_add_epi64(m2, _mm256_loadu_si256((const __m256i *)&K[56])); 669 ROUNDS4(abef, cdgh, msg); 670 m3 = _mm256_add_epi64(m3, ALIGNR1(m2, m1)); 671 m3 = _mm256_sha512msg2_epi64(m3, m2); 672 m1 = _mm256_sha512msg1_epi64(m1, _mm256_castsi256_si128(m2)); 673 674 /* Rounds 60-63 */ 675 msg = _mm256_add_epi64(m3, _mm256_loadu_si256((const __m256i *)&K[60])); 676 ROUNDS4(abef, cdgh, msg); 677 m0 = _mm256_add_epi64(m0, ALIGNR1(m3, m2)); 678 m0 = _mm256_sha512msg2_epi64(m0, m3); 679 m2 = _mm256_sha512msg1_epi64(m2, _mm256_castsi256_si128(m3)); 680 681 /* Rounds 64-67 */ 682 msg = _mm256_add_epi64(m0, _mm256_loadu_si256((const __m256i *)&K[64])); 683 ROUNDS4(abef, cdgh, msg); 684 m1 = _mm256_add_epi64(m1, ALIGNR1(m0, m3)); 685 m1 = _mm256_sha512msg2_epi64(m1, m0); 686 m3 = _mm256_sha512msg1_epi64(m3, _mm256_castsi256_si128(m0)); 687 688 /* Rounds 68-71 */ 689 msg = _mm256_add_epi64(m1, _mm256_loadu_si256((const __m256i *)&K[68])); 690 ROUNDS4(abef, cdgh, msg); 691 m2 = _mm256_add_epi64(m2, ALIGNR1(m1, m0)); 692 m2 = _mm256_sha512msg2_epi64(m2, m1); 693 694 /* Rounds 72-75 */ 695 msg = _mm256_add_epi64(m2, _mm256_loadu_si256((const __m256i *)&K[72])); 696 ROUNDS4(abef, cdgh, msg); 697 m3 = _mm256_add_epi64(m3, ALIGNR1(m2, m1)); 698 m3 = _mm256_sha512msg2_epi64(m3, m2); 699 700 /* Rounds 76-79 */ 701 msg = _mm256_add_epi64(m3, _mm256_loadu_si256((const __m256i *)&K[76])); 702 ROUNDS4(abef, cdgh, msg); 703 704 /* Add original state back */ 705 abef = _mm256_add_epi64(abef, abef_orig); 706 cdgh = _mm256_add_epi64(cdgh, cdgh_orig); 707 708 /* Restore the state's original order and store result */ 709 abef = _mm256_permute4x64_epi64(abef, 0x1b); /* f e b a */ 710 cdgh = _mm256_permute4x64_epi64(cdgh, 0x1b); /* h g d c */ 711 _mm256_storeu_si256((__m256i *)&state[0], 712 _mm256_permute2x128_si256(abef, cdgh, 0x20)); /* d c b a */ 713 _mm256_storeu_si256((__m256i *)&state[4], 714 _mm256_permute2x128_si256(abef, cdgh, 0x31)); /* h g f e */ 715 } 716 717 /* 718 * Return 1 if the CPU implements the x86 SHA512 extension and the AVX2 719 * instructions used alongside it, and the OS saves the YMM registers, 720 * 0 otherwise. 721 */ 722 int sha512_native_available(void) { 723 unsigned int eax, ebx, ecx, edx, xcr0; 724 725 if (!__get_cpuid(1, &eax, &ebx, &ecx, &edx)) 726 return 0; 727 if (!(ecx & bit_OSXSAVE) || !(ecx & bit_AVX)) 728 return 0; 729 /* XCR0 bits 1 and 2: the OS saves the XMM and YMM registers */ 730 __asm__ ("xgetbv" : "=a"(xcr0), "=d"(edx) : "c"(0)); 731 if ((xcr0 & 0x6) != 0x6) 732 return 0; 733 if (!__get_cpuid_count(7, 0, &eax, &ebx, &ecx, &edx)) 734 return 0; 735 if (eax < 1 || !(ebx & bit_AVX2)) 736 return 0; 737 __get_cpuid_count(7, 1, &eax, &ebx, &ecx, &edx); 738 return (eax & bit_SHA512) != 0; 739 } 740 741 #else 742 743 /* Stub implementations for builds without hardware support */ 744 void sha512_block_native(uint64_t *state, const uint64_t *block) { 745 (void)state; 746 (void)block; 747 /* Should never be called - use pure Haskell fallback */ 748 } 749 750 int sha512_native_available(void) { 751 return 0; 752 } 753 754 #endif