Crypto++ 8.9 Free C++ class library of cryptographic schemes

crc_simd.cpp
1 // crc_simd.cpp - written and placed in the public domain by
2 // Jeffrey Walton, Uri Blumenthal and Marcel Raad.
3 //
4 // This source file uses intrinsics to gain access to SSE4.2 and
5 // ARMv8a CRC-32 and CRC-32C instructions. A separate source file
6 // is needed because additional CXXFLAGS are required to enable
7 // the appropriate instructions sets in some build configurations.
8
9 #include "pch.h"
10 #include "config.h"
11 #include "misc.h"
12
13 #if (CRYPTOPP_SSE42_AVAILABLE)
14 # include <nmmintrin.h>
15 #endif
16
17 #if (CRYPTOPP_ARM_ACLE_HEADER)
18 # include <stdint.h>
19 # include <arm_acle.h>
20 #endif
21
22 #if (CRYPTOPP_ARM_CRC32_AVAILABLE)
23 # include "arm_simd.h"
24 #endif
25
26 #ifdef CRYPTOPP_GNU_STYLE_INLINE_ASSEMBLY
27 # include <signal.h>
28 # include <setjmp.h>
29 #endif
30
31 #if CRYPTOPP_MSC_VERSION
32 # pragma warning(disable: 4244)
33 #endif
34
35 #ifndef EXCEPTION_EXECUTE_HANDLER
36 # define EXCEPTION_EXECUTE_HANDLER 1
37 #endif
38
39 #define CONST_WORD32_CAST(x) ((const word32 *)(void*)(x))
40 #define CONST_WORD64_CAST(x) ((const word64 *)(void*)(x))
41
42 // Squash MS LNK4221 and libtool warnings
43 extern const char CRC_SIMD_FNAME[] = __FILE__;
44
45 NAMESPACE_BEGIN(CryptoPP)
46
47 #ifdef CRYPTOPP_GNU_STYLE_INLINE_ASSEMBLY
48 extern "C" {
49 typedef void (*SigHandler)(int);
50
51 static jmp_buf s_jmpSIGILL;
52 static void SigIllHandler(int)
53 {
54 longjmp(s_jmpSIGILL, 1);
55 }
56 }
57 #endif // Not CRYPTOPP_MS_STYLE_INLINE_ASSEMBLY
58
59 #if (CRYPTOPP_BOOL_ARM32 || CRYPTOPP_BOOL_ARMV8)
60
61 bool CPU_ProbeCRC32()
62 {
63 #if defined(CRYPTOPP_NO_CPU_FEATURE_PROBES)
64 return false;
65 #elif (CRYPTOPP_ARM_CRC32_AVAILABLE)
66 # if defined(CRYPTOPP_MS_STYLE_INLINE_ASSEMBLY)
67 volatile bool result = true;
68 __try
69 {
70 word32 w=0, x=1; byte z=3;
71 w = CRC32W(w,x);
72 w = CRC32B(w,z);
73 w = CRC32CW(w,x);
74 w = CRC32CB(w,z);
75
76 result = !!w;
77 }
78 __except (EXCEPTION_EXECUTE_HANDLER)
79 {
80 return false;
81 }
82 return result;
83 #else
84
85 // longjmp and clobber warnings. Volatile is required.
86 // http://github.com/weidai11/cryptopp/issues/24 and http://stackoverflow.com/q/7721854
87 volatile bool result = true;
88
89 volatile SigHandler oldHandler = signal(SIGILL, SigIllHandler);
90 if (oldHandler == SIG_ERR)
91 return false;
92
93 volatile sigset_t oldMask;
94 if (sigprocmask(0, NULLPTR, (sigset_t*)&oldMask))
95 {
96 signal(SIGILL, oldHandler);
97 return false;
98 }
99
100 if (setjmp(s_jmpSIGILL))
101 result = false;
102 else
103 {
104 word32 w=0, x=1; byte z=3;
105 w = CRC32W(w,x);
106 w = CRC32B(w,z);
107 w = CRC32CW(w,x);
108 w = CRC32CB(w,z);
109
110 // Hack... GCC optimizes away the code and returns true
111 result = !!w;
112 }
113
114 sigprocmask(SIG_SETMASK, (sigset_t*)&oldMask, NULLPTR);
115 signal(SIGILL, oldHandler);
116 return result;
117 # endif
118 #else
119 return false;
120 #endif // CRYPTOPP_ARM_CRC32_AVAILABLE
121 }
122 #endif // ARM32 or ARM64
123
124 #if (CRYPTOPP_ARM_CRC32_AVAILABLE)
125 void CRC32_Update_ARMV8(const byte *s, size_t n, word32& c)
126 {
127 for(; !IsAligned<word32>(s) && n > 0; s++, n--)
128 c = CRC32B(c, *s);
129
130 for(; n >= 16; s+=16, n-=16)
131 c = CRC32Wx4(c, CONST_WORD32_CAST(s));
132
133 for(; n >= 4; s+=4, n-=4)
134 c = CRC32W(c, *CONST_WORD32_CAST(s));
135
136 for(; n > 0; s++, n--)
137 c = CRC32B(c, *s);
138 }
139
140 void CRC32C_Update_ARMV8(const byte *s, size_t n, word32& c)
141 {
142 for(; !IsAligned<word32>(s) && n > 0; s++, n--)
143 c = CRC32CB(c, *s);
144
145 for(; n >= 16; s+=16, n-=16)
146 c = CRC32CWx4(c, CONST_WORD32_CAST(s));
147
148 for(; n >= 4; s+=4, n-=4)
149 c = CRC32CW(c, *CONST_WORD32_CAST(s));
150
151 for(; n > 0; s++, n--)
152 c = CRC32CB(c, *s);
153 }
154 #endif
155
156 #if (CRYPTOPP_SSE42_AVAILABLE)
157 void CRC32C_Update_SSE42(const byte *s, size_t n, word32& c)
158 {
159 // Temporary due to https://github.com/weidai11/cryptopp/issues/1202
160 word32 v = c;
161
162 // 64-bit code path due to https://github.com/weidai11/cryptopp/issues/1202
163 #if CRYPTOPP_BOOL_X64
164 for(; !IsAligned<word64>(s) && n > 0; s++, n--)
165 v = _mm_crc32_u8(v, *s);
166 #else
167 for(; !IsAligned<word32>(s) && n > 0; s++, n--)
168 v = _mm_crc32_u8(v, *s);
169 #endif
170
171 #if CRYPTOPP_BOOL_X64
172 for(; n >= 32; s+=32, n-=32)
173 {
174 v = _mm_crc32_u64(_mm_crc32_u64(_mm_crc32_u64(_mm_crc32_u64(v,
175 *CONST_WORD64_CAST(s+ 0)), *CONST_WORD64_CAST(s+ 8)),
176 *CONST_WORD64_CAST(s+16)), *CONST_WORD64_CAST(s+24));
177 }
178 #endif
179
180 for(; n >= 16; s+=16, n-=16)
181 {
182 v = _mm_crc32_u32(_mm_crc32_u32(_mm_crc32_u32(_mm_crc32_u32(v,
183 *CONST_WORD32_CAST(s+ 0)), *CONST_WORD32_CAST(s+ 4)),
184 *CONST_WORD32_CAST(s+ 8)), *CONST_WORD32_CAST(s+12));
185 }
186
187 for(; n >= 4; s+=4, n-=4)
188 v = _mm_crc32_u32(v, *CONST_WORD32_CAST(s));
189
190 for(; n > 0; s++, n--)
191 v = _mm_crc32_u8(v, *s);
192
193 c = static_cast<word32>(v);
194 }
195 #endif
196
197 NAMESPACE_END
arm_simd.h
Support functions for ARM and vector operations.
CRC32CWx4
uint32_t CRC32CWx4(uint32_t crc, const uint32_t vals[4])
CRC32-C checksum.
Definition: arm_simd.h:118
CRC32CB
uint32_t CRC32CB(uint32_t crc, uint8_t val)
CRC32-C checksum.
Definition: arm_simd.h:86
CRC32W
uint32_t CRC32W(uint32_t crc, uint32_t val)
CRC32 checksum.
Definition: arm_simd.h:46
CRC32B
uint32_t CRC32B(uint32_t crc, uint8_t val)
CRC32 checksum.
Definition: arm_simd.h:30
CRC32CW
uint32_t CRC32CW(uint32_t crc, uint32_t val)
CRC32-C checksum.
Definition: arm_simd.h:102
CRC32Wx4
uint32_t CRC32Wx4(uint32_t crc, const uint32_t vals[4])
CRC32 checksum.
Definition: arm_simd.h:62
config.h
Library configuration file.
word32
unsigned int word32
32-bit unsigned datatype
Definition: config_int.h:72
misc.h
Utility functions for the Crypto++ library.
CryptoPP
Crypto++ library namespace.
pch.h
Precompiled header file.