Crypto++ 7.0 Free C++ class library of cryptographic schemes

rijndael_simd.cpp
1 // rijndael_simd.cpp - written and placed in the public domain by
2 // Jeffrey Walton, Uri Blumenthal and Marcel Raad.
3 // AES-NI code originally written by Wei Dai.
4 //
5 // This source file uses intrinsics and built-ins to gain access to
6 // AES-NI, ARMv8a AES and Power8 AES instructions. A separate source
7 // file is needed because additional CXXFLAGS are required to enable
8 // the appropriate instructions sets in some build configurations.
9 //
10 // ARMv8a AES code based on CriticalBlue code from Johannes Schneiders,
11 // Skip Hovsmith and Barry O'Rourke for the mbedTLS project. Stepping
12 // mbedTLS under a debugger was helped for us to determine problems
13 // with our subkey generation and scheduling.
14 //
15 // AltiVec and Power8 code based on http://github.com/noloader/AES-Intrinsics and
16 // http://www.ibm.com/developerworks/library/se-power8-in-core-cryptography/
17 // For Power8 do not remove the casts, even when const-ness is cast away. It causes
18 // failed compiles and a 0.3 to 0.6 cpb drop in performance. The IBM documentation
19 // absolutely sucks. Thanks to Andy Polyakov, Paul R and Trudeaun for answering
20 // questions and filling the gaps in the IBM documentation.
21 //
22
23 #include "pch.h"
24 #include "config.h"
25 #include "misc.h"
26 #include "adv_simd.h"
27
28 #if (CRYPTOPP_AESNI_AVAILABLE)
29 # include <smmintrin.h>
30 # include <wmmintrin.h>
31 #endif
32
33 #if (CRYPTOPP_ARM_NEON_AVAILABLE)
34 # include <arm_neon.h>
35 #endif
36
37 #if (CRYPTOPP_ARM_ACLE_AVAILABLE)
38 # include <stdint.h>
39 # include <arm_acle.h>
40 #endif
41
42 #if defined(CRYPTOPP_POWER8_AES_AVAILABLE)
43 # include "ppc_simd.h"
44 #endif
45
46 #ifdef CRYPTOPP_GNU_STYLE_INLINE_ASSEMBLY
47 # include <signal.h>
48 # include <setjmp.h>
49 #endif
50
51 #ifndef EXCEPTION_EXECUTE_HANDLER
52 # define EXCEPTION_EXECUTE_HANDLER 1
53 #endif
54
55 // Clang __m128i casts, http://bugs.llvm.org/show_bug.cgi?id=20670
56 #define M128_CAST(x) ((__m128i *)(void *)(x))
57 #define CONST_M128_CAST(x) ((const __m128i *)(const void *)(x))
58
59 // Squash MS LNK4221 and libtool warnings
60 extern const char RIJNDAEL_SIMD_FNAME[] = __FILE__;
61
62 NAMESPACE_BEGIN(CryptoPP)
63
64 // ************************* Feature Probes ************************* //
65
66 #ifdef CRYPTOPP_GNU_STYLE_INLINE_ASSEMBLY
67 extern "C" {
68 typedef void (*SigHandler)(int);
69
70 static jmp_buf s_jmpSIGILL;
71 static void SigIllHandler(int)
72 {
73 longjmp(s_jmpSIGILL, 1);
74 }
75 }
76 #endif // Not CRYPTOPP_MS_STYLE_INLINE_ASSEMBLY
77
78 #if (CRYPTOPP_BOOL_ARM32 || CRYPTOPP_BOOL_ARMV8)
79 bool CPU_ProbeAES()
80 {
81 #if defined(CRYPTOPP_NO_CPU_FEATURE_PROBES)
82 return false;
83 #elif (CRYPTOPP_ARM_AES_AVAILABLE)
84 # if defined(CRYPTOPP_MS_STYLE_INLINE_ASSEMBLY)
85 volatile bool result = true;
86 __try
87 {
88 // AES encrypt and decrypt
89 uint8x16_t data = vdupq_n_u8(0), key = vdupq_n_u8(0);
90 uint8x16_t r1 = vaeseq_u8(data, key);
91 uint8x16_t r2 = vaesdq_u8(data, key);
92 r1 = vaesmcq_u8(r1);
93 r2 = vaesimcq_u8(r2);
94
95 result = !!(vgetq_lane_u8(r1,0) | vgetq_lane_u8(r2,7));
96 }
97 __except (EXCEPTION_EXECUTE_HANDLER)
98 {
99 return false;
100 }
101 return result;
102 # else
103
104 // longjmp and clobber warnings. Volatile is required.
105 // http://github.com/weidai11/cryptopp/issues/24 and http://stackoverflow.com/q/7721854
106 volatile bool result = true;
107
108 volatile SigHandler oldHandler = signal(SIGILL, SigIllHandler);
109 if (oldHandler == SIG_ERR)
110 return false;
111
112 volatile sigset_t oldMask;
113 if (sigprocmask(0, NULLPTR, (sigset_t*)&oldMask))
114 return false;
115
116 if (setjmp(s_jmpSIGILL))
117 result = false;
118 else
119 {
120 uint8x16_t data = vdupq_n_u8(0), key = vdupq_n_u8(0);
121 uint8x16_t r1 = vaeseq_u8(data, key);
122 uint8x16_t r2 = vaesdq_u8(data, key);
123 r1 = vaesmcq_u8(r1);
124 r2 = vaesimcq_u8(r2);
125
126 // Hack... GCC optimizes away the code and returns true
127 result = !!(vgetq_lane_u8(r1,0) | vgetq_lane_u8(r2,7));
128 }
129
130 sigprocmask(SIG_SETMASK, (sigset_t*)&oldMask, NULLPTR);
131 signal(SIGILL, oldHandler);
132 return result;
133 # endif
134 #else
135 return false;
136 #endif // CRYPTOPP_ARM_AES_AVAILABLE
137 }
138 #endif // ARM32 or ARM64
139
140 // ***************************** ARMv8 ***************************** //
141
142 #if (CRYPTOPP_ARM_AES_AVAILABLE)
143
144 ANONYMOUS_NAMESPACE_BEGIN
145
146 static inline void ARMV8_Enc_Block(uint64x2_t &data, const word32 *subkeys, unsigned int rounds)
147 {
148 CRYPTOPP_ASSERT(subkeys);
149 const byte *keys = reinterpret_cast<const byte*>(subkeys);
150 uint8x16_t block = vreinterpretq_u8_u64(data);
151
152 // AES single round encryption
153 block = vaeseq_u8(block, vld1q_u8(keys+0*16));
154 // AES mix columns
155 block = vaesmcq_u8(block);
156
157 for (unsigned int i=1; i<rounds-1; i+=2)
158 {
159 // AES single round encryption
160 block = vaeseq_u8(block, vld1q_u8(keys+i*16));
161 // AES mix columns
162 block = vaesmcq_u8(block);
163 // AES single round encryption
164 block = vaeseq_u8(block, vld1q_u8(keys+(i+1)*16));
165 // AES mix columns
166 block = vaesmcq_u8(block);
167 }
168
169 // AES single round encryption
170 block = vaeseq_u8(block, vld1q_u8(keys+(rounds-1)*16));
171 // Final Add (bitwise Xor)
172 block = veorq_u8(block, vld1q_u8(keys+rounds*16));
173
174 data = vreinterpretq_u64_u8(block);
175 }
176
177 static inline void ARMV8_Enc_6_Blocks(uint64x2_t &data0, uint64x2_t &data1,
178 uint64x2_t &data2, uint64x2_t &data3, uint64x2_t &data4, uint64x2_t &data5,
179 const word32 *subkeys, unsigned int rounds)
180 {
181 CRYPTOPP_ASSERT(subkeys);
182 const byte *keys = reinterpret_cast<const byte*>(subkeys);
183
184 uint8x16_t block0 = vreinterpretq_u8_u64(data0);
185 uint8x16_t block1 = vreinterpretq_u8_u64(data1);
186 uint8x16_t block2 = vreinterpretq_u8_u64(data2);
187 uint8x16_t block3 = vreinterpretq_u8_u64(data3);
188 uint8x16_t block4 = vreinterpretq_u8_u64(data4);
189 uint8x16_t block5 = vreinterpretq_u8_u64(data5);
190
191 uint8x16_t key;
192 for (unsigned int i=0; i<rounds-1; ++i)
193 {
194 uint8x16_t key = vld1q_u8(keys+i*16);
195 // AES single round encryption
196 block0 = vaeseq_u8(block0, key);
197 // AES mix columns
198 block0 = vaesmcq_u8(block0);
199 // AES single round encryption
200 block1 = vaeseq_u8(block1, key);
201 // AES mix columns
202 block1 = vaesmcq_u8(block1);
203 // AES single round encryption
204 block2 = vaeseq_u8(block2, key);
205 // AES mix columns
206 block2 = vaesmcq_u8(block2);
207 // AES single round encryption
208 block3 = vaeseq_u8(block3, key);
209 // AES mix columns
210 block3 = vaesmcq_u8(block3);
211 // AES single round encryption
212 block4 = vaeseq_u8(block4, key);
213 // AES mix columns
214 block4 = vaesmcq_u8(block4);
215 // AES single round encryption
216 block5 = vaeseq_u8(block5, key);
217 // AES mix columns
218 block5 = vaesmcq_u8(block5);
219 }
220
221 // AES single round encryption
222 key = vld1q_u8(keys+(rounds-1)*16);
223 block0 = vaeseq_u8(block0, key);
224 block1 = vaeseq_u8(block1, key);
225 block2 = vaeseq_u8(block2, key);
226 block3 = vaeseq_u8(block3, key);
227 block4 = vaeseq_u8(block4, key);
228 block5 = vaeseq_u8(block5, key);
229
230 // Final Add (bitwise Xor)
231 key = vld1q_u8(keys+rounds*16);
232 data0 = vreinterpretq_u64_u8(veorq_u8(block0, key));
233 data1 = vreinterpretq_u64_u8(veorq_u8(block1, key));
234 data2 = vreinterpretq_u64_u8(veorq_u8(block2, key));
235 data3 = vreinterpretq_u64_u8(veorq_u8(block3, key));
236 data4 = vreinterpretq_u64_u8(veorq_u8(block4, key));
237 data5 = vreinterpretq_u64_u8(veorq_u8(block5, key));
238 }
239
240 static inline void ARMV8_Dec_Block(uint64x2_t &data, const word32 *subkeys, unsigned int rounds)
241 {
242 CRYPTOPP_ASSERT(subkeys);
243 const byte *keys = reinterpret_cast<const byte*>(subkeys);
244 uint8x16_t block = vreinterpretq_u8_u64(data);
245
246 // AES single round decryption
247 block = vaesdq_u8(block, vld1q_u8(keys+0*16));
248 // AES inverse mix columns
249 block = vaesimcq_u8(block);
250
251 for (unsigned int i=1; i<rounds-1; i+=2)
252 {
253 // AES single round decryption
254 block = vaesdq_u8(block, vld1q_u8(keys+i*16));
255 // AES inverse mix columns
256 block = vaesimcq_u8(block);
257 // AES single round decryption
258 block = vaesdq_u8(block, vld1q_u8(keys+(i+1)*16));
259 // AES inverse mix columns
260 block = vaesimcq_u8(block);
261 }
262
263 // AES single round decryption
264 block = vaesdq_u8(block, vld1q_u8(keys+(rounds-1)*16));
265 // Final Add (bitwise Xor)
266 block = veorq_u8(block, vld1q_u8(keys+rounds*16));
267
268 data = vreinterpretq_u64_u8(block);
269 }
270
271 static inline void ARMV8_Dec_6_Blocks(uint64x2_t &data0, uint64x2_t &data1,
272 uint64x2_t &data2, uint64x2_t &data3, uint64x2_t &data4, uint64x2_t &data5,
273 const word32 *subkeys, unsigned int rounds)
274 {
275 CRYPTOPP_ASSERT(subkeys);
276 const byte *keys = reinterpret_cast<const byte*>(subkeys);
277
278 uint8x16_t block0 = vreinterpretq_u8_u64(data0);
279 uint8x16_t block1 = vreinterpretq_u8_u64(data1);
280 uint8x16_t block2 = vreinterpretq_u8_u64(data2);
281 uint8x16_t block3 = vreinterpretq_u8_u64(data3);
282 uint8x16_t block4 = vreinterpretq_u8_u64(data4);
283 uint8x16_t block5 = vreinterpretq_u8_u64(data5);
284
285 uint8x16_t key;
286 for (unsigned int i=0; i<rounds-1; ++i)
287 {
288 key = vld1q_u8(keys+i*16);
289 // AES single round decryption
290 block0 = vaesdq_u8(block0, key);
291 // AES inverse mix columns
292 block0 = vaesimcq_u8(block0);
293 // AES single round decryption
294 block1 = vaesdq_u8(block1, key);
295 // AES inverse mix columns
296 block1 = vaesimcq_u8(block1);
297 // AES single round decryption
298 block2 = vaesdq_u8(block2, key);
299 // AES inverse mix columns
300 block2 = vaesimcq_u8(block2);
301 // AES single round decryption
302 block3 = vaesdq_u8(block3, key);
303 // AES inverse mix columns
304 block3 = vaesimcq_u8(block3);
305 // AES single round decryption
306 block4 = vaesdq_u8(block4, key);
307 // AES inverse mix columns
308 block4 = vaesimcq_u8(block4);
309 // AES single round decryption
310 block5 = vaesdq_u8(block5, key);
311 // AES inverse mix columns
312 block5 = vaesimcq_u8(block5);
313 }
314
315 // AES single round decryption
316 key = vld1q_u8(keys+(rounds-1)*16);
317 block0 = vaesdq_u8(block0, key);
318 block1 = vaesdq_u8(block1, key);
319 block2 = vaesdq_u8(block2, key);
320 block3 = vaesdq_u8(block3, key);
321 block4 = vaesdq_u8(block4, key);
322 block5 = vaesdq_u8(block5, key);
323
324 // Final Add (bitwise Xor)
325 key = vld1q_u8(keys+rounds*16);
326 data0 = vreinterpretq_u64_u8(veorq_u8(block0, key));
327 data1 = vreinterpretq_u64_u8(veorq_u8(block1, key));
328 data2 = vreinterpretq_u64_u8(veorq_u8(block2, key));
329 data3 = vreinterpretq_u64_u8(veorq_u8(block3, key));
330 data4 = vreinterpretq_u64_u8(veorq_u8(block4, key));
331 data5 = vreinterpretq_u64_u8(veorq_u8(block5, key));
332 }
333
334 ANONYMOUS_NAMESPACE_END
335
336 size_t Rijndael_Enc_AdvancedProcessBlocks_ARMV8(const word32 *subKeys, size_t rounds,
337 const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
338 {
339 return AdvancedProcessBlocks128_6x1_NEON(ARMV8_Enc_Block, ARMV8_Enc_6_Blocks,
340 subKeys, rounds, inBlocks, xorBlocks, outBlocks, length, flags);
341 }
342
343 size_t Rijndael_Dec_AdvancedProcessBlocks_ARMV8(const word32 *subKeys, size_t rounds,
344 const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
345 {
346 return AdvancedProcessBlocks128_6x1_NEON(ARMV8_Dec_Block, ARMV8_Dec_6_Blocks,
347 subKeys, rounds, inBlocks, xorBlocks, outBlocks, length, flags);
348 }
349
350 #endif // CRYPTOPP_ARM_AES_AVAILABLE
351
352 // ***************************** AES-NI ***************************** //
353
354 #if (CRYPTOPP_AESNI_AVAILABLE)
355
356 ANONYMOUS_NAMESPACE_BEGIN
357
358 /* for 128-bit blocks, Rijndael never uses more than 10 rcon values */
359 CRYPTOPP_ALIGN_DATA(16)
360 const word32 s_rconLE[] = {
361 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1B, 0x36
362 };
363
364 static inline void AESNI_Enc_Block(__m128i &block, MAYBE_CONST word32 *subkeys, unsigned int rounds)
365 {
366 const __m128i* skeys = reinterpret_cast<const __m128i*>(subkeys);
367
368 block = _mm_xor_si128(block, skeys[0]);
369 for (unsigned int i=1; i<rounds-1; i+=2)
370 {
371 block = _mm_aesenc_si128(block, skeys[i]);
372 block = _mm_aesenc_si128(block, skeys[i+1]);
373 }
374 block = _mm_aesenc_si128(block, skeys[rounds-1]);
375 block = _mm_aesenclast_si128(block, skeys[rounds]);
376 }
377
378 static inline void AESNI_Enc_4_Blocks(__m128i &block0, __m128i &block1, __m128i &block2, __m128i &block3,
379 MAYBE_CONST word32 *subkeys, unsigned int rounds)
380 {
381 const __m128i* skeys = reinterpret_cast<const __m128i*>(subkeys);
382
383 __m128i rk = skeys[0];
384 block0 = _mm_xor_si128(block0, rk);
385 block1 = _mm_xor_si128(block1, rk);
386 block2 = _mm_xor_si128(block2, rk);
387 block3 = _mm_xor_si128(block3, rk);
388 for (unsigned int i=1; i<rounds; i++)
389 {
390 rk = skeys[i];
391 block0 = _mm_aesenc_si128(block0, rk);
392 block1 = _mm_aesenc_si128(block1, rk);
393 block2 = _mm_aesenc_si128(block2, rk);
394 block3 = _mm_aesenc_si128(block3, rk);
395 }
396 rk = skeys[rounds];
397 block0 = _mm_aesenclast_si128(block0, rk);
398 block1 = _mm_aesenclast_si128(block1, rk);
399 block2 = _mm_aesenclast_si128(block2, rk);
400 block3 = _mm_aesenclast_si128(block3, rk);
401 }
402
403 static inline void AESNI_Dec_Block(__m128i &block, MAYBE_CONST word32 *subkeys, unsigned int rounds)
404 {
405 const __m128i* skeys = reinterpret_cast<const __m128i*>(subkeys);
406
407 block = _mm_xor_si128(block, skeys[0]);
408 for (unsigned int i=1; i<rounds-1; i+=2)
409 {
410 block = _mm_aesdec_si128(block, skeys[i]);
411 block = _mm_aesdec_si128(block, skeys[i+1]);
412 }
413 block = _mm_aesdec_si128(block, skeys[rounds-1]);
414 block = _mm_aesdeclast_si128(block, skeys[rounds]);
415 }
416
417 static inline void AESNI_Dec_4_Blocks(__m128i &block0, __m128i &block1, __m128i &block2, __m128i &block3,
418 MAYBE_CONST word32 *subkeys, unsigned int rounds)
419 {
420 const __m128i* skeys = reinterpret_cast<const __m128i*>(subkeys);
421
422 __m128i rk = skeys[0];
423 block0 = _mm_xor_si128(block0, rk);
424 block1 = _mm_xor_si128(block1, rk);
425 block2 = _mm_xor_si128(block2, rk);
426 block3 = _mm_xor_si128(block3, rk);
427 for (unsigned int i=1; i<rounds; i++)
428 {
429 rk = skeys[i];
430 block0 = _mm_aesdec_si128(block0, rk);
431 block1 = _mm_aesdec_si128(block1, rk);
432 block2 = _mm_aesdec_si128(block2, rk);
433 block3 = _mm_aesdec_si128(block3, rk);
434 }
435 rk = skeys[rounds];
436 block0 = _mm_aesdeclast_si128(block0, rk);
437 block1 = _mm_aesdeclast_si128(block1, rk);
438 block2 = _mm_aesdeclast_si128(block2, rk);
439 block3 = _mm_aesdeclast_si128(block3, rk);
440 }
441
442 ANONYMOUS_NAMESPACE_END
443
444 void Rijndael_UncheckedSetKey_SSE4_AESNI(const byte *userKey, size_t keyLen, word32 *rk)
445 {
446 const size_t rounds = keyLen / 4 + 6;
447 const word32 *rc = s_rconLE;
448
449 __m128i temp = _mm_loadu_si128(M128_CAST(userKey+keyLen-16));
450 std::memcpy(rk, userKey, keyLen);
451
452 // keySize: m_key allocates 4*(rounds+1) word32's.
453 const size_t keySize = 4*(rounds+1);
454 const word32* end = rk + keySize;
455
456 while (true)
457 {
458 rk[keyLen/4] = rk[0] ^ _mm_extract_epi32(_mm_aeskeygenassist_si128(temp, 0), 3) ^ *(rc++);
459 rk[keyLen/4+1] = rk[1] ^ rk[keyLen/4];
460 rk[keyLen/4+2] = rk[2] ^ rk[keyLen/4+1];
461 rk[keyLen/4+3] = rk[3] ^ rk[keyLen/4+2];
462
463 if (rk + keyLen/4 + 4 == end)
464 break;
465
466 if (keyLen == 24)
467 {
468 rk[10] = rk[ 4] ^ rk[ 9];
469 rk[11] = rk[ 5] ^ rk[10];
470 temp = _mm_insert_epi32(temp, rk[11], 3);
471 }
472 else if (keyLen == 32)
473 {
474 temp = _mm_insert_epi32(temp, rk[11], 3);
475 rk[12] = rk[ 4] ^ _mm_extract_epi32(_mm_aeskeygenassist_si128(temp, 0), 2);
476 rk[13] = rk[ 5] ^ rk[12];
477 rk[14] = rk[ 6] ^ rk[13];
478 rk[15] = rk[ 7] ^ rk[14];
479 temp = _mm_insert_epi32(temp, rk[15], 3);
480 }
481 else
482 {
483 temp = _mm_insert_epi32(temp, rk[7], 3);
484 }
485
486 rk += keyLen/4;
487 }
488 }
489
490 void Rijndael_UncheckedSetKeyRev_AESNI(word32 *key, unsigned int rounds)
491 {
492 unsigned int i, j;
493 __m128i temp;
494
495 vec_swap(*M128_CAST(key), *M128_CAST(key+4*rounds));
496
497 for (i = 4, j = 4*rounds-4; i < j; i += 4, j -= 4)
498 {
499 temp = _mm_aesimc_si128(*M128_CAST(key+i));
500 *M128_CAST(key+i) = _mm_aesimc_si128(*M128_CAST(key+j));
501 *M128_CAST(key+j) = temp;
502 }
503
504 *M128_CAST(key+i) = _mm_aesimc_si128(*M128_CAST(key+i));
505 }
506
507 size_t Rijndael_Enc_AdvancedProcessBlocks_AESNI(const word32 *subKeys, size_t rounds,
508 const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
509 {
510 // SunCC workaround
511 MAYBE_CONST word32* sk = MAYBE_UNCONST_CAST(word32*, subKeys);
512 MAYBE_CONST byte* ib = MAYBE_UNCONST_CAST(byte*, inBlocks);
513 MAYBE_CONST byte* xb = MAYBE_UNCONST_CAST(byte*, xorBlocks);
514
515 return AdvancedProcessBlocks128_4x1_SSE(AESNI_Enc_Block, AESNI_Enc_4_Blocks,
516 sk, rounds, ib, xb, outBlocks, length, flags);
517 }
518
519 size_t Rijndael_Dec_AdvancedProcessBlocks_AESNI(const word32 *subKeys, size_t rounds,
520 const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
521 {
522 MAYBE_CONST word32* sk = MAYBE_UNCONST_CAST(word32*, subKeys);
523 MAYBE_CONST byte* ib = MAYBE_UNCONST_CAST(byte*, inBlocks);
524 MAYBE_CONST byte* xb = MAYBE_UNCONST_CAST(byte*, xorBlocks);
525
526 return AdvancedProcessBlocks128_4x1_SSE(AESNI_Dec_Block, AESNI_Dec_4_Blocks,
527 sk, rounds, ib, xb, outBlocks, length, flags);
528 }
529
530 #endif // CRYPTOPP_AESNI_AVAILABLE
531
532 // ************************** Power 8 Crypto ************************** //
533
534 #if (CRYPTOPP_POWER8_AES_AVAILABLE)
535
536 ANONYMOUS_NAMESPACE_BEGIN
537
538 /* for 128-bit blocks, Rijndael never uses more than 10 rcon values */
539 CRYPTOPP_ALIGN_DATA(16)
540 static const uint32_t s_rconBE[] = {
541 0x01000000, 0x02000000, 0x04000000, 0x08000000,
542 0x10000000, 0x20000000, 0x40000000, 0x80000000,
543 0x1B000000, 0x36000000
544 };
545
546 static inline void POWER8_Enc_Block(uint32x4_p &block, const word32 *subkeys, unsigned int rounds)
547 {
548 CRYPTOPP_ASSERT(IsAlignedOn(subkeys, 16));
549 const byte *keys = reinterpret_cast<const byte*>(subkeys);
550
551 uint32x4_p k = VecLoad(keys);
552 block = VecXor(block, k);
553
554 for (size_t i=1; i<rounds-1; i+=2)
555 {
556 block = VecEncrypt(block, VecLoad( i*16, keys));
557 block = VecEncrypt(block, VecLoad((i+1)*16, keys));
558 }
559
560 block = VecEncrypt(block, VecLoad((rounds-1)*16, keys));
561 block = VecEncryptLast(block, VecLoad(rounds*16, keys));
562 }
563
564 static inline void POWER8_Enc_6_Blocks(uint32x4_p &block0, uint32x4_p &block1,
565 uint32x4_p &block2, uint32x4_p &block3, uint32x4_p &block4,
566 uint32x4_p &block5, const word32 *subkeys, unsigned int rounds)
567 {
568 CRYPTOPP_ASSERT(IsAlignedOn(subkeys, 16));
569 const byte *keys = reinterpret_cast<const byte*>(subkeys);
570
571 uint32x4_p k = VecLoad(keys);
572 block0 = VecXor(block0, k);
573 block1 = VecXor(block1, k);
574 block2 = VecXor(block2, k);
575 block3 = VecXor(block3, k);
576 block4 = VecXor(block4, k);
577 block5 = VecXor(block5, k);
578
579 for (size_t i=1; i<rounds; ++i)
580 {
581 k = VecLoad(i*16, keys);
582 block0 = VecEncrypt(block0, k);
583 block1 = VecEncrypt(block1, k);
584 block2 = VecEncrypt(block2, k);
585 block3 = VecEncrypt(block3, k);
586 block4 = VecEncrypt(block4, k);
587 block5 = VecEncrypt(block5, k);
588 }
589
590 k = VecLoad(rounds*16, keys);
591 block0 = VecEncryptLast(block0, k);
592 block1 = VecEncryptLast(block1, k);
593 block2 = VecEncryptLast(block2, k);
594 block3 = VecEncryptLast(block3, k);
595 block4 = VecEncryptLast(block4, k);
596 block5 = VecEncryptLast(block5, k);
597 }
598
599 static inline void POWER8_Dec_Block(uint32x4_p &block, const word32 *subkeys, unsigned int rounds)
600 {
601 CRYPTOPP_ASSERT(IsAlignedOn(subkeys, 16));
602 const byte *keys = reinterpret_cast<const byte*>(subkeys);
603
604 uint32x4_p k = VecLoad(rounds*16, keys);
605 block = VecXor(block, k);
606
607 for (size_t i=rounds-1; i>1; i-=2)
608 {
609 block = VecDecrypt(block, VecLoad( i*16, keys));
610 block = VecDecrypt(block, VecLoad((i-1)*16, keys));
611 }
612
613 block = VecDecrypt(block, VecLoad(16, keys));
614 block = VecDecryptLast(block, VecLoad(0, keys));
615 }
616
617 static inline void POWER8_Dec_6_Blocks(uint32x4_p &block0, uint32x4_p &block1,
618 uint32x4_p &block2, uint32x4_p &block3, uint32x4_p &block4,
619 uint32x4_p &block5, const word32 *subkeys, unsigned int rounds)
620 {
621 CRYPTOPP_ASSERT(IsAlignedOn(subkeys, 16));
622 const byte *keys = reinterpret_cast<const byte*>(subkeys);
623
624 uint32x4_p k = VecLoad(rounds*16, keys);
625 block0 = VecXor(block0, k);
626 block1 = VecXor(block1, k);
627 block2 = VecXor(block2, k);
628 block3 = VecXor(block3, k);
629 block4 = VecXor(block4, k);
630 block5 = VecXor(block5, k);
631
632 for (size_t i=rounds-1; i>0; --i)
633 {
634 k = VecLoad(i*16, keys);
635 block0 = VecDecrypt(block0, k);
636 block1 = VecDecrypt(block1, k);
637 block2 = VecDecrypt(block2, k);
638 block3 = VecDecrypt(block3, k);
639 block4 = VecDecrypt(block4, k);
640 block5 = VecDecrypt(block5, k);
641 }
642
643 k = VecLoad(0, keys);
644 block0 = VecDecryptLast(block0, k);
645 block1 = VecDecryptLast(block1, k);
646 block2 = VecDecryptLast(block2, k);
647 block3 = VecDecryptLast(block3, k);
648 block4 = VecDecryptLast(block4, k);
649 block5 = VecDecryptLast(block5, k);
650 }
651
652 ANONYMOUS_NAMESPACE_END
653
654 void Rijndael_UncheckedSetKey_POWER8(const byte* userKey, size_t keyLen, word32* rk, const byte* Se)
655 {
656 const size_t rounds = keyLen / 4 + 6;
657 const word32 *rc = s_rconBE;
658 word32 *rkey = rk, temp;
659
660 GetUserKey(BIG_ENDIAN_ORDER, rkey, keyLen/4, userKey, keyLen);
661
662 // keySize: m_key allocates 4*(rounds+1) word32's.
663 const size_t keySize = 4*(rounds+1);
664 const word32* end = rkey + keySize;
665
666 while (true)
667 {
668 temp = rkey[keyLen/4-1];
669 word32 x = (word32(Se[GETBYTE(temp, 2)]) << 24) ^ (word32(Se[GETBYTE(temp, 1)]) << 16) ^
670 (word32(Se[GETBYTE(temp, 0)]) << 8) ^ Se[GETBYTE(temp, 3)];
671 rkey[keyLen/4] = rkey[0] ^ x ^ *(rc++);
672 rkey[keyLen/4+1] = rkey[1] ^ rkey[keyLen/4];
673 rkey[keyLen/4+2] = rkey[2] ^ rkey[keyLen/4+1];
674 rkey[keyLen/4+3] = rkey[3] ^ rkey[keyLen/4+2];
675
676 if (rkey + keyLen/4 + 4 == end)
677 break;
678
679 if (keyLen == 24)
680 {
681 rkey[10] = rkey[ 4] ^ rkey[ 9];
682 rkey[11] = rkey[ 5] ^ rkey[10];
683 }
684 else if (keyLen == 32)
685 {
686 temp = rkey[11];
687 rkey[12] = rkey[ 4] ^ (word32(Se[GETBYTE(temp, 3)]) << 24) ^ (word32(Se[GETBYTE(temp, 2)]) << 16) ^ (word32(Se[GETBYTE(temp, 1)]) << 8) ^ Se[GETBYTE(temp, 0)];
688 rkey[13] = rkey[ 5] ^ rkey[12];
689 rkey[14] = rkey[ 6] ^ rkey[13];
690 rkey[15] = rkey[ 7] ^ rkey[14];
691 }
692 rkey += keyLen/4;
693 }
694
695 #if (CRYPTOPP_LITTLE_ENDIAN)
696 rkey = rk;
697 const uint8x16_p mask = ((uint8x16_p){12,13,14,15, 8,9,10,11, 4,5,6,7, 0,1,2,3});
698 const uint8x16_p zero = {0};
699
700 unsigned int i=0;
701 for (i=0; i<rounds; i+=2, rkey+=8)
702 {
703 const uint8x16_p d1 = vec_vsx_ld( 0, (uint8_t*)rkey);
704 const uint8x16_p d2 = vec_vsx_ld(16, (uint8_t*)rkey);
705 vec_vsx_st(VecPermute(d1, zero, mask), 0, (uint8_t*)rkey);
706 vec_vsx_st(VecPermute(d2, zero, mask), 16, (uint8_t*)rkey);
707 }
708
709 for ( ; i<rounds+1; i++, rkey+=4)
710 {
711 const uint8x16_p d = vec_vsx_ld( 0, (uint8_t*)rkey);
712 vec_vsx_st(VecPermute(d, zero, mask), 0, (uint8_t*)rkey);
713 }
714 #endif
715 }
716
717 size_t Rijndael_Enc_AdvancedProcessBlocks128_6x1_ALTIVEC(const word32 *subKeys, size_t rounds,
718 const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
719 {
720 return AdvancedProcessBlocks128_6x1_ALTIVEC(POWER8_Enc_Block, POWER8_Enc_6_Blocks,
721 subKeys, rounds, inBlocks, xorBlocks, outBlocks, length, flags);
722 }
723
724 size_t Rijndael_Dec_AdvancedProcessBlocks128_6x1_ALTIVEC(const word32 *subKeys, size_t rounds,
725 const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
726 {
727 return AdvancedProcessBlocks128_6x1_ALTIVEC(POWER8_Dec_Block, POWER8_Dec_6_Blocks,
728 subKeys, rounds, inBlocks, xorBlocks, outBlocks, length, flags);
729 }
730
731 #endif // CRYPTOPP_POWER8_AES_AVAILABLE
732 NAMESPACE_END
VecDecryptLast
T1 VecDecryptLast(const T1 state, const T2 key)
Final round of AES decryption.
Definition: ppc_simd.h:1408
misc.h
Utility functions for the Crypto++ library.
config.h
Library configuration file.
IsAlignedOn
bool IsAlignedOn(const void *ptr, unsigned int alignment)
Determines whether ptr is aligned to a minimum value.
Definition: misc.h:1085
VecPermute
T1 VecPermute(const T1 vec, const T2 mask)
Permutes a vector.
Definition: ppc_simd.h:875
uint32x4_p
__vector unsigned int uint32x4_p
Vector of 32-bit elements.
Definition: ppc_simd.h:128
ppc_simd.h
Support functions for PowerPC and vector operations.
adv_simd.h
Template for AdvancedProcessBlocks and SIMD processing.
pch.h
Precompiled header file.
BIG_ENDIAN_ORDER
byte order is big-endian
Definition: cryptlib.h:147
CRYPTOPP_ASSERT
#define CRYPTOPP_ASSERT(exp)
Debugging and diagnostic assertion.
Definition: trap.h:60
VecXor
T1 VecXor(const T1 vec1, const T2 vec2)
XOR two vectors.
Definition: ppc_simd.h:945
VecEncryptLast
T1 VecEncryptLast(const T1 state, const T2 key)
Final round of AES encryption.
Definition: ppc_simd.h:1358
CryptoPP
Crypto++ library namespace.
VecDecrypt
T1 VecDecrypt(const T1 state, const T2 key)
One round of AES decryption.
Definition: ppc_simd.h:1383
VecLoad
uint32x4_p VecLoad(const byte src[16])
Loads a vector from a byte array.
Definition: ppc_simd.h:251
uint8x16_p
__vector unsigned char uint8x16_p
Vector of 8-bit elements.
Definition: ppc_simd.h:118
VecEncrypt
T1 VecEncrypt(const T1 state, const T2 key)
One round of AES encryption.
Definition: ppc_simd.h:1333
vec_swap
void vec_swap(T &a, T &b)
Swaps two variables which are arrays.
Definition: misc.h:527