Movatterモバイル変換

Go to the documentation of this file.

1#ifndef BLAKE3_IMPL_H

2#define BLAKE3_IMPL_H

4#include <assert.h>

5#include <stdbool.h>

6#include <stddef.h>

7#include <stdint.h>

8#include <string.h>

10#include "llvm-c/blake3.h"

11// For \p LLVM_LIBRARY_VISIBILITY

12#include "llvm/Support/Compiler.h"

14#include "llvm_blake3_prefix.h"

16// internal flags

17enumblake3_flags {

18CHUNK_START = 1 << 0,

19CHUNK_END = 1 << 1,

20PARENT = 1 << 2,

21ROOT = 1 << 3,

22KEYED_HASH = 1 << 4,

23DERIVE_KEY_CONTEXT = 1 << 5,

24DERIVE_KEY_MATERIAL = 1 << 6,

25};

27// This C implementation tries to support recent versions of GCC, Clang, and

28// MSVC.

29#if defined(_MSC_VER)

30#define INLINE static __forceinline

31#else

32#define INLINE static inline __attribute__((always_inline))

33#endif

35#if defined(__x86_64__) || defined(_M_X64)

36#define IS_X86

37#define IS_X86_64

38#endif

40#if defined(__i386__) || defined(_M_IX86)

41#define IS_X86

42#define IS_X86_32

43#endif

45#if defined(__aarch64__) || defined(_M_ARM64)

46#define IS_AARCH64

47#endif

49#if defined(IS_X86)

50#if defined(_MSC_VER)

51#include <intrin.h>

52#endif

53#include <immintrin.h>

54#endif

56#if !defined(BLAKE3_USE_NEON)

57// If BLAKE3_USE_NEON not manually set, autodetect based on

58// AArch64ness and endianness.

59 #if defined(IS_AARCH64) && !defined(__ARM_BIG_ENDIAN)

60 #define BLAKE3_USE_NEON 1

61 #else

62 #define BLAKE3_USE_NEON 0

63 #endif

64#endif

66#if defined(IS_X86)

67#define MAX_SIMD_DEGREE 16

68#elif BLAKE3_USE_NEON == 1

69#define MAX_SIMD_DEGREE 4

70#else

71#define MAX_SIMD_DEGREE 1

72#endif

74// There are some places where we want a static size that's equal to the

75// MAX_SIMD_DEGREE, but also at least 2.

76#define MAX_SIMD_DEGREE_OR_2 (MAX_SIMD_DEGREE > 2 ? MAX_SIMD_DEGREE : 2)

78staticconstuint32_t IV[8] = {0x6A09E667UL, 0xBB67AE85UL, 0x3C6EF372UL,

79 0xA54FF53AUL, 0x510E527FUL, 0x9B05688CUL,

80 0x1F83D9ABUL, 0x5BE0CD19UL};

82staticconstuint8_t MSG_SCHEDULE[7][16] = {

83 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},

84 {2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8},

85 {3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1},

86 {10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6},

87 {12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4},

88 {9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7},

89 {11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13},

90};

92/* Find index of the highest set bit */

93/* x is assumed to be nonzero. */

94staticunsignedinthighest_one(uint64_t x) {

95#if defined(__GNUC__) || defined(__clang__)

96return 63 ^ __builtin_clzll(x);

97#elif defined(_MSC_VER) && defined(IS_X86_64)

98unsignedlong index;

99 _BitScanReverse64(&index, x);

100return index;

101#elif defined(_MSC_VER) && defined(IS_X86_32)

102if(x >> 32) {

103unsignedlong index;

104 _BitScanReverse(&index, (unsignedlong)(x >> 32));

105return 32 + index;

106 }else {

107unsignedlong index;

108 _BitScanReverse(&index, (unsignedlong)x);

109return index;

110 }

111#else

112unsignedint c = 0;

113if(x & 0xffffffff00000000ULL) { x >>= 32; c += 32; }

114if(x & 0x00000000ffff0000ULL) { x >>= 16; c += 16; }

115if(x & 0x000000000000ff00ULL) { x >>= 8; c += 8; }

116if(x & 0x00000000000000f0ULL) { x >>= 4; c += 4; }

117if(x & 0x000000000000000cULL) { x >>= 2; c += 2; }

118if(x & 0x0000000000000002ULL) { c += 1; }

119return c;

120#endif

121}

122

123// Count the number of 1 bits.

124INLINEunsignedintpopcnt(uint64_t x) {

125#if defined(__GNUC__) || defined(__clang__)

126return __builtin_popcountll(x);

127#else

128unsignedint count = 0;

129while (x != 0) {

130 count += 1;

131 x &= x - 1;

132 }

133return count;

134#endif

135}

136

137// Largest power of two less than or equal to x. As a special case, returns 1

138// when x is 0.

139INLINE uint64_t round_down_to_power_of_2(uint64_t x) {

140return 1ULL <<highest_one(x | 1);

141}

142

143INLINE uint32_t counter_low(uint64_t counter) {return (uint32_t)counter; }

144

145INLINE uint32_t counter_high(uint64_t counter) {

146return (uint32_t)(counter >> 32);

147}

148

149INLINE uint32_t load32(constvoid *src) {

150constuint8_t *p = (constuint8_t *)src;

151return ((uint32_t)(p[0]) << 0) | ((uint32_t)(p[1]) << 8) |

152 ((uint32_t)(p[2]) << 16) | ((uint32_t)(p[3]) << 24);

153}

154

155INLINEvoidload_key_words(constuint8_t key[BLAKE3_KEY_LEN],

156uint32_t key_words[8]) {

157 key_words[0] =load32(&key[0 * 4]);

158 key_words[1] =load32(&key[1 * 4]);

159 key_words[2] =load32(&key[2 * 4]);

160 key_words[3] =load32(&key[3 * 4]);

161 key_words[4] =load32(&key[4 * 4]);

162 key_words[5] =load32(&key[5 * 4]);

163 key_words[6] =load32(&key[6 * 4]);

164 key_words[7] =load32(&key[7 * 4]);

165}

166

167INLINEvoidstore32(void *dst,uint32_t w) {

168uint8_t *p = (uint8_t *)dst;

169 p[0] = (uint8_t)(w >> 0);

170 p[1] = (uint8_t)(w >> 8);

171 p[2] = (uint8_t)(w >> 16);

172 p[3] = (uint8_t)(w >> 24);

173}

174

175INLINEvoidstore_cv_words(uint8_t bytes_out[32],uint32_t cv_words[8]) {

176store32(&bytes_out[0 * 4], cv_words[0]);

177store32(&bytes_out[1 * 4], cv_words[1]);

178store32(&bytes_out[2 * 4], cv_words[2]);

179store32(&bytes_out[3 * 4], cv_words[3]);

180store32(&bytes_out[4 * 4], cv_words[4]);

181store32(&bytes_out[5 * 4], cv_words[5]);

182store32(&bytes_out[6 * 4], cv_words[6]);

183store32(&bytes_out[7 * 4], cv_words[7]);

184}

185

186LLVM_LIBRARY_VISIBILITY

187voidblake3_compress_in_place(uint32_t cv[8],

188constuint8_t block[BLAKE3_BLOCK_LEN],

189uint8_t block_len,uint64_t counter,

190uint8_t flags);

191

192LLVM_LIBRARY_VISIBILITY

193voidblake3_compress_xof(constuint32_t cv[8],

194constuint8_t block[BLAKE3_BLOCK_LEN],

195uint8_t block_len,uint64_t counter,uint8_t flags,

196uint8_t out[64]);

197

198LLVM_LIBRARY_VISIBILITY

199voidblake3_hash_many(constuint8_t *const *inputs,size_t num_inputs,

200size_tblocks,constuint32_t key[8],uint64_t counter,

201bool increment_counter,uint8_t flags,

202uint8_t flags_start,uint8_t flags_end,uint8_t *out);

203

204LLVM_LIBRARY_VISIBILITY

205size_tblake3_simd_degree(void);

206

207

208// Declarations for implementation-specific functions.

209LLVM_LIBRARY_VISIBILITY

210voidblake3_compress_in_place_portable(uint32_t cv[8],

211constuint8_t block[BLAKE3_BLOCK_LEN],

212uint8_t block_len,uint64_t counter,

213uint8_t flags);

214

215LLVM_LIBRARY_VISIBILITY

216voidblake3_compress_xof_portable(constuint32_t cv[8],

217constuint8_t block[BLAKE3_BLOCK_LEN],

218uint8_t block_len,uint64_t counter,

219uint8_t flags,uint8_t out[64]);

220

221LLVM_LIBRARY_VISIBILITY

222voidblake3_hash_many_portable(constuint8_t *const *inputs,size_t num_inputs,

223size_tblocks,constuint32_t key[8],

224uint64_t counter,bool increment_counter,

225uint8_t flags,uint8_t flags_start,

226uint8_t flags_end,uint8_t *out);

227

228#if defined(IS_X86)

229#if !defined(BLAKE3_NO_SSE2)

230LLVM_LIBRARY_VISIBILITY

231voidblake3_compress_in_place_sse2(uint32_t cv[8],

232constuint8_t block[BLAKE3_BLOCK_LEN],

233uint8_t block_len,uint64_t counter,

234uint8_t flags);

235LLVM_LIBRARY_VISIBILITY

236voidblake3_compress_xof_sse2(constuint32_t cv[8],

237constuint8_t block[BLAKE3_BLOCK_LEN],

238uint8_t block_len,uint64_t counter,

239uint8_t flags,uint8_t out[64]);

240LLVM_LIBRARY_VISIBILITY

241voidblake3_hash_many_sse2(constuint8_t *const *inputs,size_t num_inputs,

242size_tblocks,constuint32_t key[8],

243uint64_t counter,bool increment_counter,

244uint8_t flags,uint8_t flags_start,

245uint8_t flags_end,uint8_t *out);

246#endif

247#if !defined(BLAKE3_NO_SSE41)

248LLVM_LIBRARY_VISIBILITY

249voidblake3_compress_in_place_sse41(uint32_t cv[8],

250constuint8_t block[BLAKE3_BLOCK_LEN],

251uint8_t block_len,uint64_t counter,

252uint8_t flags);

253LLVM_LIBRARY_VISIBILITY

254voidblake3_compress_xof_sse41(constuint32_t cv[8],

255constuint8_t block[BLAKE3_BLOCK_LEN],

256uint8_t block_len,uint64_t counter,

257uint8_t flags,uint8_t out[64]);

258LLVM_LIBRARY_VISIBILITY

259voidblake3_hash_many_sse41(constuint8_t *const *inputs,size_t num_inputs,

260size_tblocks,constuint32_t key[8],

261uint64_t counter,bool increment_counter,

262uint8_t flags,uint8_t flags_start,

263uint8_t flags_end,uint8_t *out);

264#endif

265#if !defined(BLAKE3_NO_AVX2)

266LLVM_LIBRARY_VISIBILITY

267voidblake3_hash_many_avx2(constuint8_t *const *inputs,size_t num_inputs,

268size_tblocks,constuint32_t key[8],

269uint64_t counter,bool increment_counter,

270uint8_t flags,uint8_t flags_start,

271uint8_t flags_end,uint8_t *out);

272#endif

273#if !defined(BLAKE3_NO_AVX512)

274LLVM_LIBRARY_VISIBILITY

275voidblake3_compress_in_place_avx512(uint32_t cv[8],

276constuint8_t block[BLAKE3_BLOCK_LEN],

277uint8_t block_len,uint64_t counter,

278uint8_t flags);

279

280LLVM_LIBRARY_VISIBILITY

281voidblake3_compress_xof_avx512(constuint32_t cv[8],

282constuint8_t block[BLAKE3_BLOCK_LEN],

283uint8_t block_len,uint64_t counter,

284uint8_t flags,uint8_t out[64]);

285

286LLVM_LIBRARY_VISIBILITY

287voidblake3_hash_many_avx512(constuint8_t *const *inputs,size_t num_inputs,

288size_tblocks,constuint32_t key[8],

289uint64_t counter,bool increment_counter,

290uint8_t flags,uint8_t flags_start,

291uint8_t flags_end,uint8_t *out);

292#endif

293#endif

294

295#if BLAKE3_USE_NEON == 1

296LLVM_LIBRARY_VISIBILITY

297voidblake3_hash_many_neon(constuint8_t *const *inputs,size_t num_inputs,

298size_tblocks,constuint32_t key[8],

299uint64_t counter,bool increment_counter,

300uint8_t flags,uint8_t flags_start,

301uint8_t flags_end,uint8_t *out);

302#endif

303

304

305#endif/* BLAKE3_IMPL_H */

blocks

bbsections Prepares for basic block by splitting functions into clusters of basic blocks

Definition:BasicBlockSections.cpp:140

Compiler.h

LLVM_LIBRARY_VISIBILITY

#define LLVM_LIBRARY_VISIBILITY

Definition:Compiler.h:133

block

unify loop Fixup each natural loop to have a single exit block

Definition:UnifyLoopExits.cpp:70

blake3.h

MSG_SCHEDULE

static const uint8_t MSG_SCHEDULE[7][16]

Definition:blake3_impl.h:82

popcnt

INLINE unsigned int popcnt(uint64_t x)

Definition:blake3_impl.h:124

blake3_simd_degree

LLVM_LIBRARY_VISIBILITY size_t blake3_simd_degree(void)

Definition:blake3_dispatch.c:248

INLINE

#define INLINE

Definition:blake3_impl.h:32

static const uint32_t IV[8]

Definition:blake3_impl.h:78

highest_one

static unsigned int highest_one(uint64_t x)

Definition:blake3_impl.h:94

counter_high

INLINE uint32_t counter_high(uint64_t counter)

Definition:blake3_impl.h:145

load_key_words

INLINE void load_key_words(const uint8_t key[BLAKE3_KEY_LEN], uint32_t key_words[8])

Definition:blake3_impl.h:155

store_cv_words

INLINE void store_cv_words(uint8_t bytes_out[32], uint32_t cv_words[8])

Definition:blake3_impl.h:175

round_down_to_power_of_2

INLINE uint64_t round_down_to_power_of_2(uint64_t x)

Definition:blake3_impl.h:139

load32

INLINE uint32_t load32(const void *src)

Definition:blake3_impl.h:149

store32

INLINE void store32(void *dst, uint32_t w)

Definition:blake3_impl.h:167

blake3_flags

Definition:blake3_impl.h:17

CHUNK_START

@ CHUNK_START

Definition:blake3_impl.h:18

PARENT

@ PARENT

Definition:blake3_impl.h:20

KEYED_HASH

@ KEYED_HASH

Definition:blake3_impl.h:22

DERIVE_KEY_MATERIAL

@ DERIVE_KEY_MATERIAL

Definition:blake3_impl.h:24

DERIVE_KEY_CONTEXT

@ DERIVE_KEY_CONTEXT

Definition:blake3_impl.h:23

ROOT

@ ROOT

Definition:blake3_impl.h:21

CHUNK_END

@ CHUNK_END

Definition:blake3_impl.h:19

counter_low

INLINE uint32_t counter_low(uint64_t counter)

Definition:blake3_impl.h:143

blake3_compress_in_place_sse41

#define blake3_compress_in_place_sse41

Definition:llvm_blake3_prefix.h:25

blake3_hash_many_neon

#define blake3_hash_many_neon

Definition:llvm_blake3_prefix.h:39

blake3_hash_many_avx512

#define blake3_hash_many_avx512

Definition:llvm_blake3_prefix.h:37

blake3_hash_many_avx2

#define blake3_hash_many_avx2

Definition:llvm_blake3_prefix.h:31

blake3_compress_xof_sse2

#define blake3_compress_xof_sse2

Definition:llvm_blake3_prefix.h:21

BLAKE3_BLOCK_LEN

#define BLAKE3_BLOCK_LEN

Definition:llvm_blake3_prefix.h:7

blake3_hash_many_sse41

#define blake3_hash_many_sse41

Definition:llvm_blake3_prefix.h:29

blake3_compress_xof

#define blake3_compress_xof

Definition:llvm_blake3_prefix.h:13

blake3_compress_xof_sse41

#define blake3_compress_xof_sse41

Definition:llvm_blake3_prefix.h:27

blake3_hash_many

#define blake3_hash_many

Definition:llvm_blake3_prefix.h:14

blake3_compress_in_place_sse2

#define blake3_compress_in_place_sse2

Definition:llvm_blake3_prefix.h:19

blake3_compress_in_place

#define blake3_compress_in_place

Definition:llvm_blake3_prefix.h:12

blake3_compress_xof_avx512

#define blake3_compress_xof_avx512

Definition:llvm_blake3_prefix.h:35

BLAKE3_KEY_LEN

#define BLAKE3_KEY_LEN

Definition:llvm_blake3_prefix.h:5

blake3_compress_xof_portable

#define blake3_compress_xof_portable

Definition:llvm_blake3_prefix.h:17

blake3_hash_many_portable

#define blake3_hash_many_portable

Definition:llvm_blake3_prefix.h:18

blake3_hash_many_sse2

#define blake3_hash_many_sse2

Definition:llvm_blake3_prefix.h:23

blake3_compress_in_place_portable

#define blake3_compress_in_place_portable

Definition:llvm_blake3_prefix.h:16

blake3_compress_in_place_avx512

#define blake3_compress_in_place_avx512

Definition:llvm_blake3_prefix.h:33