Crypto++ 8.9 Free C++ class library of cryptographic schemes

cpu.cpp
1 // cpu.cpp - originally written and placed in the public domain by Wei Dai
2 // modified by Jeffrey Walton and the community over the years.
3
4 #include "pch.h"
5 #include "config.h"
6
7 #ifndef EXCEPTION_EXECUTE_HANDLER
8 # define EXCEPTION_EXECUTE_HANDLER 1
9 #endif
10
11 #ifndef CRYPTOPP_IMPORTS
12
13 #include "cpu.h"
14 #include "misc.h"
15 #include "stdcpp.h"
16
17 // For _xgetbv on Microsoft 32-bit and 64-bit Intel platforms
18 // https://github.com/weidai11/cryptopp/issues/972
19 #if (CRYPTOPP_MSC_VERSION >= 1600) && (defined(_M_IX86) || defined(_M_X64))
20 # include <immintrin.h>
21 #endif
22
23 // For IsProcessorFeaturePresent on Microsoft Arm64 platforms,
24 // https://docs.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-isprocessorfeaturepresent
25 #if defined(_WIN32) && defined(_M_ARM64)
26 # include <Windows.h>
27 # include <processthreadsapi.h>
28 #endif
29
30 #ifdef _AIX
31 # include <sys/systemcfg.h>
32 #endif
33
34 #ifdef __linux__
35 # include <unistd.h>
36 #endif
37
38 // Capability queries, requires Glibc 2.16, http://lwn.net/Articles/519085/
39 // CRYPTOPP_GLIBC_VERSION not used because config.h is missing <feature.h>
40 #if (((__GLIBC__ * 100) + __GLIBC_MINOR__) >= 216)
41 # define CRYPTOPP_GETAUXV_AVAILABLE 1
42 #endif
43
44 #if CRYPTOPP_GETAUXV_AVAILABLE
45 # include <sys/auxv.h>
46 #else
47 #ifndef AT_HWCAP
48 # define AT_HWCAP 16
49 #endif
50 #ifndef AT_HWCAP2
51 # define AT_HWCAP2 26
52 #endif
53 unsigned long int getauxval(unsigned long int) { return 0; }
54 #endif
55
56 #if defined(__APPLE__)
57 # include <sys/utsname.h>
58 # include <sys/sysctl.h>
59 #endif
60
61 // FreeBSD headers are giving us trouble...
62 // https://github.com/weidai11/cryptopp/pull/1029
63 #if defined(__FreeBSD__)
64 # include <sys/auxv.h>
65 # include <sys/elf_common.h>
66 #endif
67
68 // The cpu-features header and source file are located in
69 // "$ANDROID_NDK_ROOT/sources/android/cpufeatures".
70 // setenv-android.sh will copy the header and source file
71 // into PWD and the makefile will build it in place.
72 #if defined(__ANDROID__)
73 # include "cpu-features.h"
74 #endif
75
76 #ifdef CRYPTOPP_GNU_STYLE_INLINE_ASSEMBLY
77 # include <signal.h>
78 # include <setjmp.h>
79 #endif
80
81 // Required by Visual Studio 2008 and below and Clang on Windows.
82 // Use it for all MSVC-compatible compilers.
83 // XGETBV64 and CPUID64 are in x64dll.asm.
84 #if defined(_M_X64) && defined(CRYPTOPP_MS_STYLE_INLINE_ASSEMBLY)
85 extern "C" unsigned long long __fastcall XGETBV64(unsigned int);
86 extern "C" unsigned long long __fastcall CPUID64(unsigned int, unsigned int, unsigned int*);
87 #endif
88
89 #ifdef CRYPTOPP_GNU_STYLE_INLINE_ASSEMBLY
90 extern "C" {
91 typedef void (*SigHandler)(int);
92 }
93
94 extern "C"
95 {
96 static jmp_buf s_jmpNoCPUID;
97 static void SigIllHandler(int)
98 {
99 longjmp(s_jmpNoCPUID, 1);
100 }
101 }
102 #endif // CRYPTOPP_GNU_STYLE_INLINE_ASSEMBLY
103
104 ANONYMOUS_NAMESPACE_BEGIN
105
106 #if (CRYPTOPP_BOOL_X86 || CRYPTOPP_BOOL_X32 || CRYPTOPP_BOOL_X64)
107
108 using CryptoPP::word32;
109
110 inline bool IsIntel(const word32 output[4])
111 {
112 // This is the "GenuineIntel" string
113 return (output[1] /*EBX*/ == 0x756e6547) &&
114 (output[2] /*ECX*/ == 0x6c65746e) &&
115 (output[3] /*EDX*/ == 0x49656e69);
116 }
117
118 inline bool IsAMD(const word32 output[4])
119 {
120 // This is the "AuthenticAMD" string.
121 return ((output[1] /*EBX*/ == 0x68747541) &&
122 (output[2] /*ECX*/ == 0x444D4163) &&
123 (output[3] /*EDX*/ == 0x69746E65)) ||
124 // Early K5's can return "AMDisbetter!"
125 ((output[1] /*EBX*/ == 0x69444d41) &&
126 (output[2] /*ECX*/ == 0x74656273) &&
127 (output[3] /*EDX*/ == 0x21726574));
128 }
129
130 inline bool IsHygon(const word32 output[4])
131 {
132 // This is the "HygonGenuine" string.
133 return (output[1] /*EBX*/ == 0x6f677948) &&
134 (output[2] /*ECX*/ == 0x656e6975) &&
135 (output[3] /*EDX*/ == 0x6e65476e);
136 }
137
138 inline bool IsVIA(const word32 output[4])
139 {
140 // This is the "CentaurHauls" string.
141 return ((output[1] /*EBX*/ == 0x746e6543) &&
142 (output[2] /*ECX*/ == 0x736c7561) &&
143 (output[3] /*EDX*/ == 0x48727561)) ||
144 // Some non-PadLock's return "VIA VIA VIA "
145 ((output[1] /*EBX*/ == 0x32414956) &&
146 (output[2] /*ECX*/ == 0x32414956) &&
147 (output[3] /*EDX*/ == 0x32414956));
148 }
149
150 #endif // X86, X32 and X64
151
152 #if defined(__APPLE__)
153
154 // http://stackoverflow.com/questions/45637888/how-to-determine-armv8-features-at-runtime-on-ios
155 class AppleMachineInfo
156 {
157 public:
158 enum { PowerMac=1, Mac, iPhone, iPod, iPad, AppleTV, AppleWatch };
159 enum { PowerPC=1, I386, I686, X86_64, ARM32, ARMV8, ARMV82, ARMV83 };
160
161 AppleMachineInfo() : m_device(0), m_version(0), m_arch(0)
162 {
163 struct utsname systemInfo;
164 systemInfo.machine[0] = '\0';
165 uname(&systemInfo);
166
167 std::string machine(systemInfo.machine);
168
169 std::string::size_type pos = machine.find_first_of("0123456789");
170 if (pos != std::string::npos)
171 m_version = std::atoi(machine.substr(pos).c_str());
172
173 if (machine.find("iPhone") != std::string::npos)
174 {
175 m_device = iPhone;
176 if (m_version >= 6) { m_arch = ARMV8; }
177 else { m_arch = ARM32; }
178 }
179 else if (machine.find("iPod") != std::string::npos)
180 {
181 m_device = iPod;
182 if (m_version >= 6) { m_arch = ARMV8; }
183 else { m_arch = ARM32; }
184 }
185 else if (machine.find("iPad") != std::string::npos)
186 {
187 m_device = iPad;
188 if (m_version >= 5) { m_arch = ARMV8; }
189 else { m_arch = ARM32; }
190 }
191 else if (machine.find("PowerMac") != std::string::npos ||
192 machine.find("Power Macintosh") != std::string::npos)
193 {
194 m_device = PowerMac;
195 m_arch = PowerPC;
196 }
197 else if (machine.find("Mac") != std::string::npos ||
198 machine.find("Macintosh") != std::string::npos)
199 {
200 #if defined(__x86_64) || defined(__amd64)
201 m_device = Mac;
202 m_arch = X86_64;
203 #elif defined(__i386)
204 m_device = Mac;
205 m_arch = I386;
206 #elif defined(__i686)
207 m_device = Mac;
208 m_arch = I686;
209 #else
210 // Should never get here
211 m_device = Mac;
212 m_arch = 0;
213 #endif
214 }
215 else if (machine.find("AppleTV") != std::string::npos)
216 {
217 m_device = AppleTV;
218 if (m_version >= 4) { m_arch = ARMV8; }
219 else { m_arch = ARM32; }
220 }
221 else if (machine.find("AppleWatch") != std::string::npos)
222 {
223 m_device = AppleWatch;
224 if (m_version >= 4) { m_arch = ARMV8; }
225 else { m_arch = ARM32; }
226 }
227 else if (machine.find("arm64") != std::string::npos)
228 {
229 // M1 machine?
230 std::string brand;
231 size_t size = 32;
232
233 // Supply an oversized buffer, and avoid
234 // an extra call to sysctlbyname.
235 brand.resize(size);
236 if (sysctlbyname("machdep.cpu.brand_string", &brand[0], &size, NULL, 0) == 0 && size > 0)
237 {
238 if (brand[size-1] == '\0')
239 size--;
240 brand.resize(size);
241 }
242
243 if (brand == "Apple M1")
244 {
245 m_device = Mac;
246 m_arch = ARMV82;
247 }
248 else
249 {
250 // ???
251 m_device = 0;
252 m_arch = ARMV8;
253 }
254 }
255 else
256 {
257 CRYPTOPP_ASSERT(0);
258 }
259 }
260
261 unsigned int Device() const {
262 return m_device;
263 }
264
265 unsigned int Version() const {
266 return m_version;
267 }
268
269 unsigned int Arch() const {
270 return m_arch;
271 }
272
273 bool IsARM32() const {
274 return m_arch == ARM32;
275 }
276
277 bool IsARMv8() const {
278 return m_arch >= ARMV8;
279 }
280
281 bool IsARMv82() const {
282 return m_arch >= ARMV82;
283 }
284
285 bool IsARMv83() const {
286 return m_arch >= ARMV83;
287 }
288
289 private:
290 unsigned int m_device, m_version, m_arch;
291 };
292
293 void GetAppleMachineInfo(unsigned int& device, unsigned int& version, unsigned int& arch)
294 {
295 #if CRYPTOPP_CXX11_STATIC_INIT
296 static const AppleMachineInfo info;
297 #else
298 using CryptoPP::Singleton;
299 const AppleMachineInfo& info = Singleton<AppleMachineInfo>().Ref();
300 #endif
301
302 device = info.Device();
303 version = info.Version();
304 arch = info.Arch();
305 }
306
307 inline bool IsAppleMachineARM32()
308 {
309 static unsigned int arch;
310 if (arch == 0)
311 {
312 unsigned int unused;
313 GetAppleMachineInfo(unused, unused, arch);
314 }
315 return arch == AppleMachineInfo::ARM32;
316 }
317
318 inline bool IsAppleMachineARMv8()
319 {
320 static unsigned int arch;
321 if (arch == 0)
322 {
323 unsigned int unused;
324 GetAppleMachineInfo(unused, unused, arch);
325 }
326 return arch >= AppleMachineInfo::ARMV8;
327 }
328
329 inline bool IsAppleMachineARMv82()
330 {
331 static unsigned int arch;
332 if (arch == 0)
333 {
334 unsigned int unused;
335 GetAppleMachineInfo(unused, unused, arch);
336 }
337 return arch >= AppleMachineInfo::ARMV82;
338 }
339
340 inline bool IsAppleMachineARMv83()
341 {
342 static unsigned int arch;
343 if (arch == 0)
344 {
345 unsigned int unused;
346 GetAppleMachineInfo(unused, unused, arch);
347 }
348 return arch >= AppleMachineInfo::ARMV83;
349 }
350
351 #endif // __APPLE__
352
353 ANONYMOUS_NAMESPACE_END
354
355 NAMESPACE_BEGIN(CryptoPP)
356
357 // *************************** IA-32 CPUs ***************************
358
359 #if (CRYPTOPP_BOOL_X86 || CRYPTOPP_BOOL_X32 || CRYPTOPP_BOOL_X64)
360
361 bool CRYPTOPP_SECTION_INIT g_x86DetectionDone = false;
362 bool CRYPTOPP_SECTION_INIT g_hasSSE2 = false;
363 bool CRYPTOPP_SECTION_INIT g_hasSSSE3 = false;
364 bool CRYPTOPP_SECTION_INIT g_hasSSE41 = false;
365 bool CRYPTOPP_SECTION_INIT g_hasSSE42 = false;
366 bool CRYPTOPP_SECTION_INIT g_hasAESNI = false;
367 bool CRYPTOPP_SECTION_INIT g_hasCLMUL = false;
368 bool CRYPTOPP_SECTION_INIT g_hasMOVBE = false;
369 bool CRYPTOPP_SECTION_INIT g_hasAVX = false;
370 bool CRYPTOPP_SECTION_INIT g_hasAVX2 = false;
371 bool CRYPTOPP_SECTION_INIT g_hasADX = false;
372 bool CRYPTOPP_SECTION_INIT g_hasSHA = false;
373 bool CRYPTOPP_SECTION_INIT g_hasRDRAND = false;
374 bool CRYPTOPP_SECTION_INIT g_hasRDSEED = false;
375 bool CRYPTOPP_SECTION_INIT g_isP4 = false;
376 bool CRYPTOPP_SECTION_INIT g_hasPadlockRNG = false;
377 bool CRYPTOPP_SECTION_INIT g_hasPadlockACE = false;
378 bool CRYPTOPP_SECTION_INIT g_hasPadlockACE2 = false;
379 bool CRYPTOPP_SECTION_INIT g_hasPadlockPHE = false;
380 bool CRYPTOPP_SECTION_INIT g_hasPadlockPMM = false;
381 word32 CRYPTOPP_SECTION_INIT g_cacheLineSize = CRYPTOPP_L1_CACHE_LINE_SIZE;
382
383 // For Solaris 11
384 extern bool CPU_ProbeSSE2();
385
386 // xcr0 is available when xgetbv is present.
387 // The intrinsic is broke on GCC 8.1 and earlier. Also see
388 // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=85684.
389 word64 XGetBV(word32 num)
390 {
391 // Explicitly handle CRYPTOPP_DISABLE_ASM case.
392 // https://github.com/weidai11/cryptopp/issues/1240
393 #if defined(CRYPTOPP_DISABLE_ASM)
394 return 0;
395
396 // Required by Visual Studio 2008 and below and Clang on Windows.
397 // Use it for all MSVC-compatible compilers.
398 #elif defined(_M_X64) && defined(CRYPTOPP_MS_STYLE_INLINE_ASSEMBLY)
399
400 return XGETBV64(num);
401
402 // Required by Visual Studio 2008 and below and Clang on Windows.
403 // Use it for all MSVC-compatible compilers.
404 #elif defined(_M_IX86) && defined(CRYPTOPP_MS_STYLE_INLINE_ASSEMBLY)
405
406 word32 a=0, d=0;
407 __asm {
408 push eax
409 push edx
410 push ecx
411 mov ecx, num
412 _emit 0x0f
413 _emit 0x01
414 _emit 0xd0
415 mov a, eax
416 mov d, edx
417 pop ecx
418 pop edx
419 pop eax
420 }
421 return (static_cast<word64>(d) << 32) | a;
422
423 // GCC 4.4 and above
424 #elif (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 4))
425
426 word32 a=0, d=0;
427 __asm__
428 (
429 "xgetbv" : "=a"(a), "=d"(d) : "c"(num) : "cc"
430 );
431 return (static_cast<word64>(d) << 32) | a;
432
433 // Remainder of GCC and compatibles.
434 #elif defined(__GNUC__) || defined(__clang__) || defined(__SUNPRO_CC)
435
436 // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=71659 and
437 // http://www.agner.org/optimize/vectorclass/read.php?i=65
438 word32 a=0, d=0;
439 __asm__
440 (
441 ".byte 0x0f, 0x01, 0xd0" "\n\t"
442 : "=a"(a), "=d"(d) : "c"(num) : "cc"
443 );
444 return (static_cast<word64>(d) << 32) | a;
445 #else
446 # error "Need an xgetbv function"
447 #endif
448 }
449
450 // No inline due to Borland/Embarcadero and Issue 498
451 // cpu.cpp (131): E2211 Inline assembly not allowed in inline and template functions
452 bool CpuId(word32 func, word32 subfunc, word32 output[4])
453 {
454 // Explicitly handle CRYPTOPP_DISABLE_ASM case.
455 // https://github.com/weidai11/cryptopp/issues/1240
456 #if defined(CRYPTOPP_DISABLE_ASM)
457 output[0] = output[1] = output[2] = output[3] = 0;
458 return false;
459
460 // Required by Visual Studio 2008 and below and Clang on Windows.
461 // Use it for all MSVC-compatible compilers.
462 #elif defined(_M_X64) && defined(CRYPTOPP_MS_STYLE_INLINE_ASSEMBLY)
463
464 CPUID64(func, subfunc, output);
465 return true;
466
467 // Required by Visual Studio 2008 and below and Clang on Windows.
468 // Use it for all MSVC-compatible compilers.
469 #elif defined(_M_IX86) && defined(CRYPTOPP_MS_STYLE_INLINE_ASSEMBLY)
470
471 __try
472 {
473 // Borland/Embarcadero and Issue 500
474 // Local variables for cpuid output
475 word32 a, b, c, d;
476 __asm
477 {
478 push ebx
479 mov eax, func
480 mov ecx, subfunc
481 cpuid
482 mov [a], eax
483 mov [b], ebx
484 mov [c], ecx
485 mov [d], edx
486 pop ebx
487 }
488 output[0] = a;
489 output[1] = b;
490 output[2] = c;
491 output[3] = d;
492 }
493 __except (EXCEPTION_EXECUTE_HANDLER)
494 {
495 return false;
496 }
497
498 return true;
499
500 // Linux, Unix, OS X, Solaris, Cygwin, MinGW
501 #else
502
503 // longjmp and clobber warnings. Volatile is required.
504 // http://github.com/weidai11/cryptopp/issues/24 and http://stackoverflow.com/q/7721854
505 volatile bool result = true;
506
507 volatile SigHandler oldHandler = signal(SIGILL, SigIllHandler);
508 if (oldHandler == SIG_ERR)
509 return false;
510
511 # ifndef __MINGW32__
512 volatile sigset_t oldMask;
513 if (sigprocmask(0, NULLPTR, (sigset_t*)&oldMask) != 0)
514 {
515 signal(SIGILL, oldHandler);
516 return false;
517 }
518 # endif
519
520 if (setjmp(s_jmpNoCPUID))
521 result = false;
522 else
523 {
524 asm volatile
525 (
526 // save ebx in case -fPIC is being used
527 # if CRYPTOPP_BOOL_X32 || CRYPTOPP_BOOL_X64
528 "pushq %%rbx; cpuid; mov %%ebx, %%edi; popq %%rbx"
529 # else
530 "push %%ebx; cpuid; mov %%ebx, %%edi; pop %%ebx"
531 # endif
532 : "=a" (output[0]), "=D" (output[1]), "=c" (output[2]), "=d" (output[3])
533 : "a" (func), "c" (subfunc)
534 : "cc"
535 );
536 }
537
538 # ifndef __MINGW32__
539 sigprocmask(SIG_SETMASK, (sigset_t*)&oldMask, NULLPTR);
540 # endif
541
542 signal(SIGILL, oldHandler);
543 return result;
544 #endif
545 }
546
547 void DetectX86Features()
548 {
549 // Coverity finding CID 171239. Initialize arrays.
550 // Indexes: EAX=0, EBX=1, ECX=2, EDX=3
551 word32 cpuid0[4]={0}, cpuid1[4]={0}, cpuid2[4]={0};
552
553 #if defined(CRYPTOPP_DISABLE_ASM)
554 // Not available
555 goto done;
556 #else
557 if (!CpuId(0, 0, cpuid0))
558 goto done;
559 if (!CpuId(1, 0, cpuid1))
560 goto done;
561 #endif
562
563 CRYPTOPP_CONSTANT(EAX_REG = 0);
564 CRYPTOPP_CONSTANT(EBX_REG = 1);
565 CRYPTOPP_CONSTANT(ECX_REG = 2);
566 CRYPTOPP_CONSTANT(EDX_REG = 3);
567
568 CRYPTOPP_CONSTANT(MMX_FLAG = (1 << 24)); // EDX
569 CRYPTOPP_CONSTANT(SSE_FLAG = (1 << 25)); // EDX
570 CRYPTOPP_CONSTANT(SSE2_FLAG = (1 << 26)); // EDX
571
572 CRYPTOPP_CONSTANT(SSE3_FLAG = (1 << 0)); // ECX
573 CRYPTOPP_CONSTANT(SSSE3_FLAG = (1 << 9)); // ECX
574 CRYPTOPP_CONSTANT(SSE41_FLAG = (1 << 19)); // ECX
575 CRYPTOPP_CONSTANT(SSE42_FLAG = (1 << 20)); // ECX
576 CRYPTOPP_CONSTANT(MOVBE_FLAG = (1 << 22)); // ECX
577 CRYPTOPP_CONSTANT(AESNI_FLAG = (1 << 25)); // ECX
578 CRYPTOPP_CONSTANT(CLMUL_FLAG = (1 << 1)); // ECX
579
580 CRYPTOPP_CONSTANT(XSAVE_FLAG = (1 << 26)); // ECX
581 CRYPTOPP_CONSTANT(OSXSAVE_FLAG = (1 << 27)); // ECX
582
583 CRYPTOPP_CONSTANT(AVX_FLAG = (3 << 27)); // ECX
584 CRYPTOPP_CONSTANT(YMM_FLAG = (3 << 1)); // CR0
585
586 // x86_64 machines don't check some flags because SSE2
587 // is part of the core instruction set architecture
588 CRYPTOPP_UNUSED(MMX_FLAG); CRYPTOPP_UNUSED(SSE_FLAG);
589 CRYPTOPP_UNUSED(SSE2_FLAG); CRYPTOPP_UNUSED(SSE3_FLAG);
590 CRYPTOPP_UNUSED(XSAVE_FLAG);
591
592 #if (CRYPTOPP_BOOL_X32 || CRYPTOPP_BOOL_X64)
593 // 64-bit core instruction set includes SSE2. Just check
594 // the OS enabled SSE2 support using OSXSAVE.
595 g_hasSSE2 = (cpuid1[ECX_REG] & OSXSAVE_FLAG) != 0;
596 #else
597 // Check the processor supports SSE2. Then use OSXSAVE to
598 // signal OS support for SSE2 to avoid probes.
599 // Also see http://stackoverflow.com/a/22521619/608639
600 // and http://github.com/weidai11/cryptopp/issues/511.
601 if ((cpuid1[EDX_REG] & SSE2_FLAG) == SSE2_FLAG)
602 g_hasSSE2 = (cpuid1[ECX_REG] & XSAVE_FLAG) != 0 &&
603 (cpuid1[ECX_REG] & OSXSAVE_FLAG) != 0;
604 #endif
605
606 // Solaris 11 i86pc does not signal SSE support using
607 // OSXSAVE. Additionally, Fedora 38 on a 2015 Celeron
608 // N3700 does not set OSXSAVE. So we need to explicitly
609 // probe for SSE support on rare occasions. Ugh...
610 if (g_hasSSE2 == false)
611 {
612 g_hasSSE2 = CPU_ProbeSSE2();
613 if (g_hasSSE2 == false)
614 goto done;
615 }
616
617 g_hasSSSE3 = (cpuid1[ECX_REG] & SSSE3_FLAG) != 0;
618 g_hasSSE41 = (cpuid1[ECX_REG] & SSE41_FLAG) != 0;
619 g_hasSSE42 = (cpuid1[ECX_REG] & SSE42_FLAG) != 0;
620 g_hasMOVBE = (cpuid1[ECX_REG] & MOVBE_FLAG) != 0;
621 g_hasAESNI = (cpuid1[ECX_REG] & AESNI_FLAG) != 0;
622 g_hasCLMUL = (cpuid1[ECX_REG] & CLMUL_FLAG) != 0;
623
624 // AVX is similar to SSE. Check if AVX is available on the cpu, then
625 // check if the OS enabled XSAVE/XRESTORE for the extended registers.
626 // https://software.intel.com/en-us/blogs/2011/04/14/is-avx-enabled
627 if ((cpuid1[ECX_REG] & AVX_FLAG) == AVX_FLAG)
628 {
629 word64 xcr0 = XGetBV(0);
630 g_hasAVX = (xcr0 & YMM_FLAG) == YMM_FLAG;
631 }
632
633 if (IsIntel(cpuid0))
634 {
635 CRYPTOPP_CONSTANT(RDRAND_FLAG = (1 << 30));
636 CRYPTOPP_CONSTANT(RDSEED_FLAG = (1 << 18));
637 CRYPTOPP_CONSTANT( ADX_FLAG = (1 << 19));
638 CRYPTOPP_CONSTANT( SHA_FLAG = (1 << 29));
639 CRYPTOPP_CONSTANT( AVX2_FLAG = (1 << 5));
640
641 g_isP4 = ((cpuid1[0] >> 8) & 0xf) == 0xf;
642 g_cacheLineSize = 8 * GETBYTE(cpuid1[1], 1);
643 g_hasRDRAND = (cpuid1[ECX_REG] & RDRAND_FLAG) != 0;
644
645 if (cpuid0[EAX_REG] >= 7)
646 {
647 if (CpuId(7, 0, cpuid2))
648 {
649 g_hasRDSEED = (cpuid2[EBX_REG] & RDSEED_FLAG) != 0;
650 g_hasADX = (cpuid2[EBX_REG] & ADX_FLAG) != 0;
651 g_hasSHA = (cpuid2[EBX_REG] & SHA_FLAG) != 0;
652 g_hasAVX2 = (cpuid2[EBX_REG] & AVX2_FLAG) != 0;
653 }
654 }
655 }
656 else if (IsAMD(cpuid0) || IsHygon(cpuid0))
657 {
658 CRYPTOPP_CONSTANT(RDRAND_FLAG = (1 << 30));
659 CRYPTOPP_CONSTANT(RDSEED_FLAG = (1 << 18));
660 CRYPTOPP_CONSTANT( ADX_FLAG = (1 << 19));
661 CRYPTOPP_CONSTANT( SHA_FLAG = (1 << 29));
662 CRYPTOPP_CONSTANT( AVX2_FLAG = (1 << 5));
663
664 CpuId(0x80000005, 0, cpuid2);
665 g_cacheLineSize = GETBYTE(cpuid2[ECX_REG], 0);
666 g_hasRDRAND = (cpuid1[ECX_REG] & RDRAND_FLAG) != 0;
667
668 if (cpuid0[EAX_REG] >= 7)
669 {
670 if (CpuId(7, 0, cpuid2))
671 {
672 g_hasRDSEED = (cpuid2[EBX_REG] & RDSEED_FLAG) != 0;
673 g_hasADX = (cpuid2[EBX_REG] & ADX_FLAG) != 0;
674 g_hasSHA = (cpuid2[EBX_REG] & SHA_FLAG) != 0;
675 g_hasAVX2 = (cpuid2[EBX_REG] & AVX2_FLAG) != 0;
676 }
677 }
678
679 // Unconditionally disable RDRAND and RDSEED on AMD cpu's with family 15h or 16h.
680 // See Crypto++ Issue 924, https://github.com/weidai11/cryptopp/issues/924,
681 // Clear RDRAND CPUID bit on AMD family 15h/16h, https://lore.kernel.org/patchwork/patch/1115413/,
682 // and AMD CPUID Specification, https://www.amd.com/system/files/TechDocs/25481.pdf
683 {
684 CRYPTOPP_CONSTANT(FAMILY_BASE_FLAG = (0x0f << 8));
685 CRYPTOPP_CONSTANT(FAMILY_EXT_FLAG = (0xff << 20));
686
687 word32 family = (cpuid1[0] & FAMILY_BASE_FLAG) >> 8;
688 if (family == 0xf)
689 family += (cpuid1[0] & FAMILY_EXT_FLAG) >> 20;
690 if (family == 0x15 || family == 0x16)
691 {
692 g_hasRDRAND = false;
693 g_hasRDSEED = false;
694 }
695 }
696 }
697 else if (IsVIA(cpuid0))
698 {
699 // Two bits: available and enabled
700 CRYPTOPP_CONSTANT( RNG_FLAGS = (0x3 << 2));
701 CRYPTOPP_CONSTANT( ACE_FLAGS = (0x3 << 6));
702 CRYPTOPP_CONSTANT(ACE2_FLAGS = (0x3 << 8));
703 CRYPTOPP_CONSTANT( PHE_FLAGS = (0x3 << 10));
704 CRYPTOPP_CONSTANT( PMM_FLAGS = (0x3 << 12));
705
706 CpuId(0xC0000000, 0, cpuid2);
707 word32 extendedFeatures = cpuid2[0];
708
709 if (extendedFeatures >= 0xC0000001)
710 {
711 CpuId(0xC0000001, 0, cpuid2);
712 g_hasPadlockRNG = (cpuid2[EDX_REG] & RNG_FLAGS) != 0;
713 g_hasPadlockACE = (cpuid2[EDX_REG] & ACE_FLAGS) != 0;
714 g_hasPadlockACE2 = (cpuid2[EDX_REG] & ACE2_FLAGS) != 0;
715 g_hasPadlockPHE = (cpuid2[EDX_REG] & PHE_FLAGS) != 0;
716 g_hasPadlockPMM = (cpuid2[EDX_REG] & PMM_FLAGS) != 0;
717 }
718
719 if (extendedFeatures >= 0xC0000005)
720 {
721 CpuId(0xC0000005, 0, cpuid2);
722 g_cacheLineSize = GETBYTE(cpuid2[ECX_REG], 0);
723 }
724 }
725
726 // Keep AVX2 in sync with OS support for AVX. AVX tests both
727 // cpu support and OS support, while AVX2 only tests cpu support.
728 g_hasAVX2 &= g_hasAVX;
729
730 done:
731
732 #if defined(_SC_LEVEL1_DCACHE_LINESIZE)
733 // Glibc does not implement on some platforms. The runtime returns 0 instead of error.
734 // https://sourceware.org/git/?p=glibc.git;a=blob;f=sysdeps/posix/sysconf.c
735 int cacheLineSize = (int)sysconf(_SC_LEVEL1_DCACHE_LINESIZE);
736 if (g_cacheLineSize == 0 && cacheLineSize > 0)
737 g_cacheLineSize = cacheLineSize;
738 #endif
739
740 if (g_cacheLineSize == 0)
741 g_cacheLineSize = CRYPTOPP_L1_CACHE_LINE_SIZE;
742
743 *const_cast<volatile bool*>(&g_x86DetectionDone) = true;
744 }
745
746 // *************************** ARM-32, Aarch32 and Aarch64 ***************************
747
748 #elif (CRYPTOPP_BOOL_ARM32 || CRYPTOPP_BOOL_ARMV8)
749
750 bool CRYPTOPP_SECTION_INIT g_ArmDetectionDone = false;
751 bool CRYPTOPP_SECTION_INIT g_hasARMv7 = false;
752 bool CRYPTOPP_SECTION_INIT g_hasNEON = false;
753 bool CRYPTOPP_SECTION_INIT g_hasPMULL = false;
754 bool CRYPTOPP_SECTION_INIT g_hasCRC32 = false;
755 bool CRYPTOPP_SECTION_INIT g_hasAES = false;
756 bool CRYPTOPP_SECTION_INIT g_hasSHA1 = false;
757 bool CRYPTOPP_SECTION_INIT g_hasSHA2 = false;
758 bool CRYPTOPP_SECTION_INIT g_hasSHA512 = false;
759 bool CRYPTOPP_SECTION_INIT g_hasSHA3 = false;
760 bool CRYPTOPP_SECTION_INIT g_hasSM3 = false;
761 bool CRYPTOPP_SECTION_INIT g_hasSM4 = false;
762 word32 CRYPTOPP_SECTION_INIT g_cacheLineSize = CRYPTOPP_L1_CACHE_LINE_SIZE;
763
764 // ARM does not have an unprivileged equivalent to CPUID on IA-32. We have to
765 // jump through some hoops to detect features on a wide array of platforms.
766 // Our strategy is two part. First, attempt to *Query* the OS for a feature,
767 // like using getauxval on Linux. If that fails, then *Probe* the cpu
768 // executing an instruction and an observe a SIGILL if unsupported. The probes
769 // are in source files where compilation options like -march=armv8-a+crc make
770 // intrinsics available. They are expensive when compared to a standard OS
771 // feature query. Always perform the feature query first. For Linux see
772 // http://sourceware.org/ml/libc-help/2017-08/msg00012.html
773 // Avoid probes on Apple platforms because Apple's signal handling for SIGILLs
774 // appears broken. We are trying to figure out a way to feature test without
775 // probes. Also see http://stackoverflow.com/a/11197770/608639 and
776 // http://gist.github.com/erkanyildiz/390a480f27e86f8cd6ba.
777
778 extern bool CPU_ProbeARMv7();
779 extern bool CPU_ProbeNEON();
780 extern bool CPU_ProbeCRC32();
781 extern bool CPU_ProbeAES();
782 extern bool CPU_ProbeSHA1();
783 extern bool CPU_ProbeSHA256();
784 extern bool CPU_ProbeSHA512();
785 extern bool CPU_ProbeSHA3();
786 extern bool CPU_ProbeSM3();
787 extern bool CPU_ProbeSM4();
788 extern bool CPU_ProbePMULL();
789
790 // https://github.com/torvalds/linux/blob/master/arch/arm/include/uapi/asm/hwcap.h
791 // https://github.com/torvalds/linux/blob/master/arch/arm64/include/uapi/asm/hwcap.h
792 #ifndef HWCAP_ARMv7
793 # define HWCAP_ARMv7 (1 << 29)
794 #endif
795 #ifndef HWCAP_ASIMD
796 # define HWCAP_ASIMD (1 << 1)
797 #endif
798 #ifndef HWCAP_NEON
799 # define HWCAP_NEON (1 << 12)
800 #endif
801 #ifndef HWCAP_CRC32
802 # define HWCAP_CRC32 (1 << 7)
803 #endif
804 #ifndef HWCAP2_CRC32
805 # define HWCAP2_CRC32 (1 << 4)
806 #endif
807 #ifndef HWCAP_PMULL
808 # define HWCAP_PMULL (1 << 4)
809 #endif
810 #ifndef HWCAP2_PMULL
811 # define HWCAP2_PMULL (1 << 1)
812 #endif
813 #ifndef HWCAP_AES
814 # define HWCAP_AES (1 << 3)
815 #endif
816 #ifndef HWCAP2_AES
817 # define HWCAP2_AES (1 << 0)
818 #endif
819 #ifndef HWCAP_SHA1
820 # define HWCAP_SHA1 (1 << 5)
821 #endif
822 #ifndef HWCAP_SHA2
823 # define HWCAP_SHA2 (1 << 6)
824 #endif
825 #ifndef HWCAP2_SHA1
826 # define HWCAP2_SHA1 (1 << 2)
827 #endif
828 #ifndef HWCAP2_SHA2
829 # define HWCAP2_SHA2 (1 << 3)
830 #endif
831 #ifndef HWCAP_SHA3
832 # define HWCAP_SHA3 (1 << 17)
833 #endif
834 #ifndef HWCAP_SM3
835 # define HWCAP_SM3 (1 << 18)
836 #endif
837 #ifndef HWCAP_SM4
838 # define HWCAP_SM4 (1 << 19)
839 #endif
840 #ifndef HWCAP_SHA512
841 # define HWCAP_SHA512 (1 << 21)
842 #endif
843
844 inline bool CPU_QueryARMv7()
845 {
846 #if defined(__ANDROID__) && defined(__arm__)
847 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM) != 0) &&
848 ((android_getCpuFeatures() & ANDROID_CPU_ARM_FEATURE_ARMv7) != 0))
849 return true;
850 #elif defined(__linux__) && defined(__arm__)
851 if ((getauxval(AT_HWCAP) & HWCAP_ARMv7) != 0 ||
852 (getauxval(AT_HWCAP) & HWCAP_NEON) != 0)
853 return true;
854 #elif defined(__APPLE__) && defined(__arm__)
855 // Apple hardware is ARMv7 or above.
856 return true;
857 #elif defined(_WIN32) && defined(_M_ARM64)
858 // Windows 10 ARM64 is only supported on Armv8a and above
859 return true;
860 #endif
861 return false;
862 }
863
864 inline bool CPU_QueryNEON()
865 {
866 #if defined(__ANDROID__) && defined(__aarch64__)
867 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM64) != 0) &&
868 ((android_getCpuFeatures() & ANDROID_CPU_ARM64_FEATURE_ASIMD) != 0))
869 return true;
870 #elif defined(__ANDROID__) && defined(__arm__)
871 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM) != 0) &&
872 ((android_getCpuFeatures() & ANDROID_CPU_ARM_FEATURE_NEON) != 0))
873 return true;
874 #elif defined(__linux__) && defined(__aarch64__)
875 if ((getauxval(AT_HWCAP) & HWCAP_ASIMD) != 0)
876 return true;
877 #elif defined(__linux__) && defined(__aarch32__)
878 if ((getauxval(AT_HWCAP2) & HWCAP2_ASIMD) != 0)
879 return true;
880 #elif defined(__linux__) && defined(__arm__)
881 if ((getauxval(AT_HWCAP) & HWCAP_NEON) != 0)
882 return true;
883 #elif defined(__APPLE__) && defined(__aarch64__)
884 // Core feature set for Aarch32 and Aarch64.
885 if (IsAppleMachineARMv8())
886 return true;
887 #elif defined(_WIN32) && defined(_M_ARM64)
888 // Windows 10 ARM64 is only supported on Armv8a and above
889 if (IsProcessorFeaturePresent(PF_ARM_V8_INSTRUCTIONS_AVAILABLE) != 0)
890 return true;
891 #endif
892 return false;
893 }
894
895 inline bool CPU_QueryCRC32()
896 {
897 #if defined(__ANDROID__) && defined(__aarch64__)
898 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM64) != 0) &&
899 ((android_getCpuFeatures() & ANDROID_CPU_ARM64_FEATURE_CRC32) != 0))
900 return true;
901 #elif defined(__ANDROID__) && defined(__aarch32__)
902 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM) != 0) &&
903 ((android_getCpuFeatures() & ANDROID_CPU_ARM_FEATURE_CRC32) != 0))
904 return true;
905 #elif defined(__linux__) && defined(__aarch64__)
906 if ((getauxval(AT_HWCAP) & HWCAP_CRC32) != 0)
907 return true;
908 #elif defined(__linux__) && defined(__aarch32__)
909 if ((getauxval(AT_HWCAP2) & HWCAP2_CRC32) != 0)
910 return true;
911 #elif defined(__APPLE__) && defined(__aarch64__)
912 // M1 processor
913 if (IsAppleMachineARMv82())
914 return true;
915 #elif defined(_WIN32) && defined(_M_ARM64)
916 if (IsProcessorFeaturePresent(PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE) != 0)
917 return true;
918 #endif
919 return false;
920 }
921
922 inline bool CPU_QueryPMULL()
923 {
924 #if defined(__ANDROID__) && defined(__aarch64__)
925 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM64) != 0) &&
926 ((android_getCpuFeatures() & ANDROID_CPU_ARM64_FEATURE_PMULL) != 0))
927 return true;
928 #elif defined(__ANDROID__) && defined(__aarch32__)
929 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM) != 0) &&
930 ((android_getCpuFeatures() & ANDROID_CPU_ARM_FEATURE_PMULL) != 0))
931 return true;
932 #elif defined(__linux__) && defined(__aarch64__)
933 if ((getauxval(AT_HWCAP) & HWCAP_PMULL) != 0)
934 return true;
935 #elif defined(__linux__) && defined(__aarch32__)
936 if ((getauxval(AT_HWCAP2) & HWCAP2_PMULL) != 0)
937 return true;
938 #elif defined(__APPLE__) && defined(__aarch64__)
939 // M1 processor
940 if (IsAppleMachineARMv82())
941 return true;
942 #elif defined(_WIN32) && defined(_M_ARM64)
943 if (IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE) != 0)
944 return true;
945 #endif
946 return false;
947 }
948
949 inline bool CPU_QueryAES()
950 {
951 #if defined(__ANDROID__) && defined(__aarch64__)
952 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM64) != 0) &&
953 ((android_getCpuFeatures() & ANDROID_CPU_ARM64_FEATURE_AES) != 0))
954 return true;
955 #elif defined(__ANDROID__) && defined(__aarch32__)
956 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM) != 0) &&
957 ((android_getCpuFeatures() & ANDROID_CPU_ARM_FEATURE_AES) != 0))
958 return true;
959 #elif defined(__linux__) && defined(__aarch64__)
960 if ((getauxval(AT_HWCAP) & HWCAP_AES) != 0)
961 return true;
962 #elif defined(__linux__) && defined(__aarch32__)
963 if ((getauxval(AT_HWCAP2) & HWCAP2_AES) != 0)
964 return true;
965 #elif defined(__APPLE__) && defined(__aarch64__)
966 // M1 processor
967 if (IsAppleMachineARMv82())
968 return true;
969 #elif defined(_WIN32) && defined(_M_ARM64)
970 if (IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE) != 0)
971 return true;
972 #endif
973 return false;
974 }
975
976 inline bool CPU_QuerySHA1()
977 {
978 #if defined(__ANDROID__) && defined(__aarch64__)
979 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM64) != 0) &&
980 ((android_getCpuFeatures() & ANDROID_CPU_ARM64_FEATURE_SHA1) != 0))
981 return true;
982 #elif defined(__ANDROID__) && defined(__aarch32__)
983 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM) != 0) &&
984 ((android_getCpuFeatures() & ANDROID_CPU_ARM_FEATURE_SHA1) != 0))
985 return true;
986 #elif defined(__linux__) && defined(__aarch64__)
987 if ((getauxval(AT_HWCAP) & HWCAP_SHA1) != 0)
988 return true;
989 #elif defined(__linux__) && defined(__aarch32__)
990 if ((getauxval(AT_HWCAP2) & HWCAP2_SHA1) != 0)
991 return true;
992 #elif defined(__APPLE__) && defined(__aarch64__)
993 // M1 processor
994 if (IsAppleMachineARMv82())
995 return true;
996 #elif defined(_WIN32) && defined(_M_ARM64)
997 if (IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE) != 0)
998 return true;
999 #endif
1000 return false;
1001 }
1002
1003 inline bool CPU_QuerySHA256()
1004 {
1005 #if defined(__ANDROID__) && defined(__aarch64__)
1006 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM64) != 0) &&
1007 ((android_getCpuFeatures() & ANDROID_CPU_ARM64_FEATURE_SHA2) != 0))
1008 return true;
1009 #elif defined(__ANDROID__) && defined(__aarch32__)
1010 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM) != 0) &&
1011 ((android_getCpuFeatures() & ANDROID_CPU_ARM_FEATURE_SHA2) != 0))
1012 return true;
1013 #elif defined(__linux__) && defined(__aarch64__)
1014 if ((getauxval(AT_HWCAP) & HWCAP_SHA2) != 0)
1015 return true;
1016 #elif defined(__linux__) && defined(__aarch32__)
1017 if ((getauxval(AT_HWCAP2) & HWCAP2_SHA2) != 0)
1018 return true;
1019 #elif defined(__APPLE__) && defined(__aarch64__)
1020 // M1 processor
1021 if (IsAppleMachineARMv82())
1022 return true;
1023 #elif defined(_WIN32) && defined(_M_ARM64)
1024 if (IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE) != 0)
1025 return true;
1026 #endif
1027 return false;
1028 }
1029
1030 // Some ARMv8.2 features are disabled at the moment
1031 inline bool CPU_QuerySHA3()
1032 {
1033 // According to the ARM manual, SHA3 depends upon SHA1 and SHA2.
1034 // If SHA1 and SHA2 are not present, then SHA3 and SHA512 are
1035 // not present. Also see Arm A64 Instruction Set Architecture,
1036 // https://developer.arm.com/documentation/ddi0596/2020-12/
1037 if (!g_hasSHA1 || !g_hasSHA2) { return false; }
1038
1039 #if defined(__ANDROID__) && defined(__aarch64__) && 0
1040 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM64) != 0) &&
1041 ((android_getCpuFeatures() & ANDROID_CPU_ARM64_FEATURE_SHA3) != 0))
1042 return true;
1043 #elif defined(__ANDROID__) && defined(__aarch32__) && 0
1044 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM) != 0) &&
1045 ((android_getCpuFeatures() & ANDROID_CPU_ARM_FEATURE_SHA3) != 0))
1046 return true;
1047 #elif defined(__linux__) && defined(__aarch64__)
1048 if ((getauxval(AT_HWCAP) & HWCAP_SHA3) != 0)
1049 return true;
1050 #elif defined(__linux__) && defined(__aarch32__)
1051 if ((getauxval(AT_HWCAP2) & HWCAP2_SHA3) != 0)
1052 return true;
1053 #elif defined(__APPLE__) && defined(__aarch64__)
1054 // M1 processor
1055 if (IsAppleMachineARMv82())
1056 return true;
1057 #endif
1058 return false;
1059 }
1060
1061 // Some ARMv8.2 features are disabled at the moment
1062 inline bool CPU_QuerySHA512()
1063 {
1064 // According to the ARM manual, SHA512 depends upon SHA1 and SHA2.
1065 // If SHA1 and SHA2 are not present, then SHA3 and SHA512 are
1066 // not present. Also see Arm A64 Instruction Set Architecture,
1067 // https://developer.arm.com/documentation/ddi0596/2020-12/
1068 if (!g_hasSHA1 || !g_hasSHA2) { return false; }
1069
1070 #if defined(__ANDROID__) && defined(__aarch64__) && 0
1071 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM64) != 0) &&
1072 ((android_getCpuFeatures() & ANDROID_CPU_ARM64_FEATURE_SHA512) != 0))
1073 return true;
1074 #elif defined(__ANDROID__) && defined(__aarch32__) && 0
1075 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM) != 0) &&
1076 ((android_getCpuFeatures() & ANDROID_CPU_ARM_FEATURE_SHA512) != 0))
1077 return true;
1078 #elif defined(__linux__) && defined(__aarch64__)
1079 if ((getauxval(AT_HWCAP) & HWCAP_SHA512) != 0)
1080 return true;
1081 #elif defined(__linux__) && defined(__aarch32__)
1082 if ((getauxval(AT_HWCAP2) & HWCAP2_SHA512) != 0)
1083 return true;
1084 #elif defined(__APPLE__) && defined(__aarch64__)
1085 // M1 processor
1086 if (IsAppleMachineARMv82())
1087 return true;
1088 #endif
1089 return false;
1090 }
1091
1092 // Some ARMv8.2 features are disabled at the moment
1093 inline bool CPU_QuerySM3()
1094 {
1095 #if defined(__ANDROID__) && defined(__aarch64__) && 0
1096 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM64) != 0) &&
1097 ((android_getCpuFeatures() & ANDROID_CPU_ARM64_FEATURE_SM3) != 0))
1098 return true;
1099 #elif defined(__ANDROID__) && defined(__aarch32__) && 0
1100 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM) != 0) &&
1101 ((android_getCpuFeatures() & ANDROID_CPU_ARM_FEATURE_SM3) != 0))
1102 return true;
1103 #elif defined(__linux__) && defined(__aarch64__)
1104 if ((getauxval(AT_HWCAP) & HWCAP_SM3) != 0)
1105 return true;
1106 #elif defined(__linux__) && defined(__aarch32__)
1107 if ((getauxval(AT_HWCAP2) & HWCAP2_SM3) != 0)
1108 return true;
1109 #elif defined(__APPLE__) && defined(__aarch64__) && 0
1110 // No Apple support yet.
1111 #endif
1112 return false;
1113 }
1114
1115 // Some ARMv8.2 features are disabled at the moment
1116 inline bool CPU_QuerySM4()
1117 {
1118 #if defined(__ANDROID__) && defined(__aarch64__) && 0
1119 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM64) != 0) &&
1120 ((android_getCpuFeatures() & ANDROID_CPU_ARM64_FEATURE_SM4) != 0))
1121 return true;
1122 #elif defined(__ANDROID__) && defined(__aarch32__) && 0
1123 if (((android_getCpuFamily() & ANDROID_CPU_FAMILY_ARM) != 0) &&
1124 ((android_getCpuFeatures() & ANDROID_CPU_ARM_FEATURE_SM4) != 0))
1125 return true;
1126 #elif defined(__linux__) && defined(__aarch64__)
1127 if ((getauxval(AT_HWCAP) & HWCAP_SM4) != 0)
1128 return true;
1129 #elif defined(__linux__) && defined(__aarch32__)
1130 if ((getauxval(AT_HWCAP2) & HWCAP2_SM4) != 0)
1131 return true;
1132 #elif defined(__APPLE__) && defined(__aarch64__) && 0
1133 // No Apple support yet.
1134 #endif
1135 return false;
1136 }
1137
1138 void DetectArmFeatures()
1139 {
1140 #ifndef CRYPTOPP_DISABLE_ASM
1141
1142 // The CPU_ProbeXXX's return false for OSes which
1143 // can't tolerate SIGILL-based probes
1144 g_hasARMv7 = CPU_QueryARMv7() || CPU_ProbeARMv7();
1145 g_hasNEON = CPU_QueryNEON() || CPU_ProbeNEON();
1146 g_hasCRC32 = CPU_QueryCRC32() || CPU_ProbeCRC32();
1147 g_hasPMULL = CPU_QueryPMULL() || CPU_ProbePMULL();
1148 g_hasAES = CPU_QueryAES() || CPU_ProbeAES();
1149 g_hasSHA1 = CPU_QuerySHA1() || CPU_ProbeSHA1();
1150 g_hasSHA2 = CPU_QuerySHA256() || CPU_ProbeSHA256();
1151 g_hasSHA512 = CPU_QuerySHA512(); // || CPU_ProbeSHA512();
1152 g_hasSHA3 = CPU_QuerySHA3(); // || CPU_ProbeSHA3();
1153 g_hasSM3 = CPU_QuerySM3(); // || CPU_ProbeSM3();
1154 g_hasSM4 = CPU_QuerySM4(); // || CPU_ProbeSM4();
1155
1156 #if defined(_SC_LEVEL1_DCACHE_LINESIZE)
1157 // Glibc does not implement on some platforms. The runtime returns 0 instead of error.
1158 // https://sourceware.org/git/?p=glibc.git;a=blob;f=sysdeps/posix/sysconf.c
1159 int cacheLineSize = (int)sysconf(_SC_LEVEL1_DCACHE_LINESIZE);
1160 if (cacheLineSize > 0)
1161 g_cacheLineSize = cacheLineSize;
1162 #endif
1163
1164 if (g_cacheLineSize == 0)
1165 g_cacheLineSize = CRYPTOPP_L1_CACHE_LINE_SIZE;
1166
1167 #endif // CRYPTOPP_DISABLE_ASM
1168
1169 *const_cast<volatile bool*>(&g_ArmDetectionDone) = true;
1170 }
1171
1172 // *************************** PowerPC and PowerPC64 ***************************
1173
1174 #elif (CRYPTOPP_BOOL_PPC32 || CRYPTOPP_BOOL_PPC64)
1175
1176 bool CRYPTOPP_SECTION_INIT g_PowerPcDetectionDone = false;
1177 bool CRYPTOPP_SECTION_INIT g_hasAltivec = false;
1178 bool CRYPTOPP_SECTION_INIT g_hasPower7 = false;
1179 bool CRYPTOPP_SECTION_INIT g_hasPower8 = false;
1180 bool CRYPTOPP_SECTION_INIT g_hasPower9 = false;
1181 bool CRYPTOPP_SECTION_INIT g_hasAES = false;
1182 bool CRYPTOPP_SECTION_INIT g_hasPMULL = false;
1183 bool CRYPTOPP_SECTION_INIT g_hasSHA256 = false;
1184 bool CRYPTOPP_SECTION_INIT g_hasSHA512 = false;
1185 bool CRYPTOPP_SECTION_INIT g_hasDARN = false;
1186 word32 CRYPTOPP_SECTION_INIT g_cacheLineSize = CRYPTOPP_L1_CACHE_LINE_SIZE;
1187
1188 extern bool CPU_ProbeAltivec();
1189 extern bool CPU_ProbePower7();
1190 extern bool CPU_ProbePower8();
1191 extern bool CPU_ProbePower9();
1192 extern bool CPU_ProbeAES();
1193 extern bool CPU_ProbePMULL();
1194 extern bool CPU_ProbeSHA256();
1195 extern bool CPU_ProbeSHA512();
1196 extern bool CPU_ProbeDARN();
1197
1198 // AIX defines. We used to just call __power_7_andup()
1199 // and friends but at Power9, too many compilers were
1200 // missing __power_9_andup(). Instead we switched to
1201 // a pattern similar to OpenSSL caps testing.
1202 #ifndef __power_6_andup
1203 # define __power_6_andup() __power_set(0xffffffffU<<14)
1204 #endif
1205 #ifndef __power_7_andup
1206 # define __power_7_andup() __power_set(0xffffffffU<<15)
1207 #endif
1208 #ifndef __power_8_andup
1209 # define __power_8_andup() __power_set(0xffffffffU<<16)
1210 #endif
1211 #ifndef __power_9_andup
1212 # define __power_9_andup() __power_set(0xffffffffU<<17)
1213 #endif
1214
1215 // AIX first supported Altivec at Power6, though it
1216 // was available much earlier for other vendors.
1217 inline bool CPU_QueryAltivec()
1218 {
1219 #if defined(__linux__) && defined(PPC_FEATURE_HAS_ALTIVEC)
1220 if ((getauxval(AT_HWCAP) & PPC_FEATURE_HAS_ALTIVEC) != 0)
1221 return true;
1222 #elif defined(_AIX)
1223 if (__power_6_andup() != 0)
1224 return true;
1225 #elif defined(__APPLE__) && defined(__POWERPC__)
1226 unsigned int unused, arch;
1227 GetAppleMachineInfo(unused, unused, arch);
1228 return arch == AppleMachineInfo::PowerMac;
1229 #elif defined(__FreeBSD__) && defined(PPC_FEATURE_HAS_ALTIVEC)
1230 unsigned long cpufeatures;
1231 if (elf_aux_info(AT_HWCAP, &cpufeatures, sizeof(cpufeatures)) == 0)
1232 if ((cpufeatures & PPC_FEATURE_HAS_ALTIVEC) != 0)
1233 return true;
1234 #endif
1235 return false;
1236 }
1237
1238 inline bool CPU_QueryPower7()
1239 {
1240 // Power7 and ISA 2.06
1241 #if defined(__linux__) && defined(PPC_FEATURE_ARCH_2_06)
1242 if ((getauxval(AT_HWCAP) & PPC_FEATURE_ARCH_2_06) != 0)
1243 return true;
1244 #elif defined(_AIX)
1245 if (__power_7_andup() != 0)
1246 return true;
1247 #elif defined(__FreeBSD__) && defined(PPC_FEATURE_ARCH_2_06)
1248 unsigned long cpufeatures;
1249 if (elf_aux_info(AT_HWCAP, &cpufeatures, sizeof(cpufeatures)) == 0)
1250 if ((cpufeatures & PPC_FEATURE_ARCH_2_06) != 0)
1251 return true;
1252 #endif
1253 return false;
1254 }
1255
1256 inline bool CPU_QueryPower8()
1257 {
1258 // Power8 and ISA 2.07 provide in-core crypto.
1259 #if defined(__linux__) && defined(PPC_FEATURE2_ARCH_2_07)
1260 if ((getauxval(AT_HWCAP2) & PPC_FEATURE2_ARCH_2_07) != 0)
1261 return true;
1262 #elif defined(_AIX)
1263 if (__power_8_andup() != 0)
1264 return true;
1265 #elif defined(__FreeBSD__) && defined(PPC_FEATURE2_ARCH_2_07)
1266 unsigned long cpufeatures;
1267 if (elf_aux_info(AT_HWCAP, &cpufeatures, sizeof(cpufeatures)) == 0)
1268 if ((cpufeatures & PPC_FEATURE_ARCH_2_07) != 0)
1269 return true;
1270 #endif
1271 return false;
1272 }
1273
1274 inline bool CPU_QueryPower9()
1275 {
1276 // Power9 and ISA 3.0.
1277 #if defined(__linux__) && defined(PPC_FEATURE2_ARCH_3_00)
1278 if ((getauxval(AT_HWCAP2) & PPC_FEATURE2_ARCH_3_00) != 0)
1279 return true;
1280 #elif defined(_AIX)
1281 if (__power_9_andup() != 0)
1282 return true;
1283 #elif defined(__FreeBSD__) && defined(PPC_FEATURE2_ARCH_3_00)
1284 unsigned long cpufeatures;
1285 if (elf_aux_info(AT_HWCAP, &cpufeatures, sizeof(cpufeatures)) == 0)
1286 if ((cpufeatures & PPC_FEATURE_ARCH2_3_00) != 0)
1287 return true;
1288 #endif
1289 return false;
1290 }
1291
1292 inline bool CPU_QueryAES()
1293 {
1294 // Power8 and ISA 2.07 provide in-core crypto. Glibc
1295 // 2.24 or higher is required for PPC_FEATURE2_VEC_CRYPTO.
1296 #if defined(__linux__) && defined(PPC_FEATURE2_VEC_CRYPTO)
1297 if ((getauxval(AT_HWCAP2) & PPC_FEATURE2_VEC_CRYPTO) != 0)
1298 return true;
1299 #elif defined(_AIX)
1300 if (__power_8_andup() != 0)
1301 return true;
1302 #elif defined(__FreeBSD__) && defined(PPC_FEATURE2_HAS_VEC_CRYPTO)
1303 unsigned long cpufeatures;
1304 if (elf_aux_info(AT_HWCAP2, &cpufeatures, sizeof(cpufeatures)) == 0)
1305 if ((cpufeatures & PPC_FEATURE2_HAS_VEC_CRYPTO != 0)
1306 return true;
1307 #endif
1308 return false;
1309 }
1310
1311 inline bool CPU_QueryPMULL()
1312 {
1313 // Power8 and ISA 2.07 provide in-core crypto. Glibc
1314 // 2.24 or higher is required for PPC_FEATURE2_VEC_CRYPTO.
1315 #if defined(__linux__) && defined(PPC_FEATURE2_VEC_CRYPTO)
1316 if ((getauxval(AT_HWCAP2) & PPC_FEATURE2_VEC_CRYPTO) != 0)
1317 return true;
1318 #elif defined(_AIX)
1319 if (__power_8_andup() != 0)
1320 return true;
1321 #elif defined(__FreeBSD__) && defined(PPC_FEATURE2_HAS_VEC_CRYPTO)
1322 unsigned long cpufeatures;
1323 if (elf_aux_info(AT_HWCAP2, &cpufeatures, sizeof(cpufeatures)) == 0)
1324 if ((cpufeatures & PPC_FEATURE2_HAS_VEC_CRYPTO != 0)
1325 return true;
1326 #endif
1327 return false;
1328 }
1329
1330 inline bool CPU_QuerySHA256()
1331 {
1332 // Power8 and ISA 2.07 provide in-core crypto. Glibc
1333 // 2.24 or higher is required for PPC_FEATURE2_VEC_CRYPTO.
1334 #if defined(__linux__) && defined(PPC_FEATURE2_VEC_CRYPTO)
1335 if ((getauxval(AT_HWCAP2) & PPC_FEATURE2_VEC_CRYPTO) != 0)
1336 return true;
1337 #elif defined(_AIX)
1338 if (__power_8_andup() != 0)
1339 return true;
1340 #elif defined(__FreeBSD__) && defined(PPC_FEATURE2_HAS_VEC_CRYPTO)
1341 unsigned long cpufeatures;
1342 if (elf_aux_info(AT_HWCAP2, &cpufeatures, sizeof(cpufeatures)) == 0)
1343 if ((cpufeatures & PPC_FEATURE2_HAS_VEC_CRYPTO != 0)
1344 return true;
1345 #endif
1346 return false;
1347 }
1348 inline bool CPU_QuerySHA512()
1349 {
1350 // Power8 and ISA 2.07 provide in-core crypto. Glibc
1351 // 2.24 or higher is required for PPC_FEATURE2_VEC_CRYPTO.
1352 #if defined(__linux__) && defined(PPC_FEATURE2_VEC_CRYPTO)
1353 if ((getauxval(AT_HWCAP2) & PPC_FEATURE2_VEC_CRYPTO) != 0)
1354 return true;
1355 #elif defined(_AIX)
1356 if (__power_8_andup() != 0)
1357 return true;
1358 #elif defined(__FreeBSD__) && defined(PPC_FEATURE2_HAS_VEC_CRYPTO)
1359 unsigned long cpufeatures;
1360 if (elf_aux_info(AT_HWCAP2, &cpufeatures, sizeof(cpufeatures)) == 0)
1361 if ((cpufeatures & PPC_FEATURE2_HAS_VEC_CRYPTO != 0)
1362 return true;
1363 #endif
1364 return false;
1365 }
1366
1367 // Power9 random number generator
1368 inline bool CPU_QueryDARN()
1369 {
1370 // Power9 and ISA 3.0 provide DARN. It looks like
1371 // Glibc offers PPC_FEATURE2_DARN.
1372 #if defined(__linux__) && defined(PPC_FEATURE2_ARCH_3_00)
1373 if ((getauxval(AT_HWCAP2) & PPC_FEATURE2_ARCH_3_00) != 0)
1374 return true;
1375 #elif defined(_AIX)
1376 if (__power_9_andup() != 0)
1377 return true;
1378 #elif defined(__FreeBSD__) && defined(PPC_FEATURE2_ARCH_3_00)
1379 unsigned long cpufeatures;
1380 if (elf_aux_info(AT_HWCAP2, &cpufeatures, sizeof(cpufeatures)) == 0)
1381 if ((cpufeatures & PPC_FEATURE2_ARCH_3_00) != 0)
1382 return true;
1383 #endif
1384 return false;
1385 }
1386
1387 void DetectPowerPcFeatures()
1388 {
1389 // GCC 10 is giving us trouble in CPU_ProbePower9() and CPU_ProbeDARN().
1390 // GCC is generating POWER9 instructions on POWER8 for ppc_power9.cpp.
1391 // The compiler idiots did not think through the consequences of
1392 // requiring us to use -mcpu=power9 to unlock the ISA. Epic fail.
1393 // https://github.com/weidai11/cryptopp/issues/986
1394
1395 #ifndef CRYPTOPP_DISABLE_ASM
1396
1397 // The CPU_ProbeXXX's return false for OSes which
1398 // can't tolerate SIGILL-based probes, like Apple
1399 g_hasAltivec = CPU_QueryAltivec() || CPU_ProbeAltivec();
1400 g_hasPower7 = CPU_QueryPower7() || CPU_ProbePower7();
1401 g_hasPower8 = CPU_QueryPower8() || CPU_ProbePower8();
1402 g_hasPower9 = CPU_QueryPower9() || CPU_ProbePower9();
1403 g_hasPMULL = CPU_QueryPMULL() || CPU_ProbePMULL();
1404 g_hasAES = CPU_QueryAES() || CPU_ProbeAES();
1405 g_hasSHA256 = CPU_QuerySHA256() || CPU_ProbeSHA256();
1406 g_hasSHA512 = CPU_QuerySHA512() || CPU_ProbeSHA512();
1407 g_hasDARN = CPU_QueryDARN() || CPU_ProbeDARN();
1408
1409 #if defined(_AIX) && defined(SC_L1C_DLS)
1410 // /usr/include/sys/systemcfg.h
1411 int cacheLineSize = getsystemcfg(SC_L1C_DLS);
1412 if (cacheLineSize > 0)
1413 g_cacheLineSize = cacheLineSize;
1414 #elif defined(_SC_LEVEL1_DCACHE_LINESIZE)
1415 // Glibc does not implement on some platforms. The runtime returns 0 instead of error.
1416 // https://sourceware.org/git/?p=glibc.git;a=blob;f=sysdeps/posix/sysconf.c
1417 int cacheLineSize = (int)sysconf(_SC_LEVEL1_DCACHE_LINESIZE);
1418 if (cacheLineSize > 0)
1419 g_cacheLineSize = cacheLineSize;
1420 #endif
1421
1422 if (g_cacheLineSize == 0)
1423 g_cacheLineSize = CRYPTOPP_L1_CACHE_LINE_SIZE;
1424
1425 #endif // CRYPTOPP_DISABLE_ASM
1426
1427 *const_cast<volatile bool*>(&g_PowerPcDetectionDone) = true;
1428 }
1429
1430 #endif
1431 NAMESPACE_END
1432
1433 // *************************** C++ Static Initialization ***************************
1434
1435 ANONYMOUS_NAMESPACE_BEGIN
1436
1437 class InitCpu
1438 {
1439 public:
1440 InitCpu()
1441 {
1442 #if CRYPTOPP_BOOL_X86 || CRYPTOPP_BOOL_X32 || CRYPTOPP_BOOL_X64
1443 CryptoPP::DetectX86Features();
1444 #elif CRYPTOPP_BOOL_ARM32 || CRYPTOPP_BOOL_ARMV8
1445 CryptoPP::DetectArmFeatures();
1446 #elif CRYPTOPP_BOOL_PPC32 || CRYPTOPP_BOOL_PPC64
1447 CryptoPP::DetectPowerPcFeatures();
1448 #endif
1449 }
1450 };
1451
1452 // This is not really needed because HasSSE() and friends can dynamically initialize.
1453 // Everything depends on CPU features so we initialize it once at load time.
1454 // Dynamic initialization will be used if init priorities are not available.
1455
1456 #if HAVE_GCC_INIT_PRIORITY
1457 const InitCpu s_init __attribute__ ((init_priority (CRYPTOPP_INIT_PRIORITY + 10))) = InitCpu();
1458 #elif HAVE_MSC_INIT_PRIORITY
1459 #pragma warning(disable: 4075)
1460 #pragma init_seg(".CRT$XCU")
1461 const InitCpu s_init;
1462 #pragma warning(default: 4075)
1463 #elif HAVE_XLC_INIT_PRIORITY
1464 // XLC needs constant, not a define
1465 #pragma priority(270)
1466 const InitCpu s_init;
1467 #else
1468 const InitCpu s_init;
1469 #endif
1470
1471 ANONYMOUS_NAMESPACE_END
1472
1473 #endif // CRYPTOPP_IMPORTS
Singleton
Restricts the instantiation of a class to one static object without locks.
Definition: misc.h:309
Singleton::Ref
const T & Ref(...) const
Return a reference to the inner Singleton object.
Definition: misc.h:329
config.h
Library configuration file.
CRYPTOPP_L1_CACHE_LINE_SIZE
#define CRYPTOPP_L1_CACHE_LINE_SIZE
L1 data cache line size.
Definition: config_cpu.h:147
CRYPTOPP_SECTION_INIT
#define CRYPTOPP_SECTION_INIT
Initialized data section.
Definition: config_cpu.h:168
word32
unsigned int word32
32-bit unsigned datatype
Definition: config_int.h:72
word64
unsigned long long word64
64-bit unsigned datatype
Definition: config_int.h:101
cpu.h
Functions for CPU features and intrinsics.
misc.h
Utility functions for the Crypto++ library.
CryptoPP
Crypto++ library namespace.
Name::Version
const char * Version()
int
Definition: argnames.h:18
pch.h
Precompiled header file.
stdcpp.h
Common C++ header files.
CRYPTOPP_ASSERT
#define CRYPTOPP_ASSERT(exp)
Debugging and diagnostic assertion.
Definition: trap.h:68