- Main Page
- Namespaces
- Classes
- Files

# gcm.cpp

    00001 // gcm.cpp - written and placed in the public domain by Wei Dai
    00002
    00003 // use "cl /EP /P /DCRYPTOPP_GENERATE_X64_MASM gcm.cpp" to generate MASM code
    00004
    00005 #include "pch.h"
    00006
    00007 #ifndef CRYPTOPP_IMPORTS
    00008 #ifndef CRYPTOPP_GENERATE_X64_MASM
    00009
    00010 #include "gcm.h"
    00011 #include "cpu.h"
    00012
    00013 #if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE
    00014 #include <emmintrin.h>
    00015 #endif
    00016
    00017 NAMESPACE_BEGIN(CryptoPP)
    00018
    00019 word16 GCM_Base::s_reductionTable[256];
    00020 bool GCM_Base::s_reductionTableInitialized = false;
    00021
    00022 void GCM_Base::GCTR::IncrementCounterBy256()
    00023 {
    00024         IncrementCounterByOne(m_counterArray+BlockSize()-4, 3);
    00025 }
    00026
    00027 #if 0
    00028 // preserved for testing
    00029 void gcm_gf_mult(const unsigned char *a, const unsigned char *b, unsigned char *c)
    00030 {
    00031         word64 Z0=0, Z1=0, V0, V1;
    00032
    00033         typedef BlockGetAndPut<word64, BigEndian> Block;
    00034         Block::Get(a)(V0)(V1);
    00035
    00036         for (int i=0; i<16; i++)
    00037         {
    00038                 for (int j=0x80; j!=0; j>>=1)
    00039                 {
    00040                         int x = b[i] & j;
    00041                         Z0 ^= x ? V0 : 0;
    00042                         Z1 ^= x ? V1 : 0;
    00043                         x = (int)V1 & 1;
    00044                         V1 = (V1>>1) | (V0<<63);
    00045                         V0 = (V0>>1) ^ (x ? W64LIT(0xe1) << 56 : 0);
    00046                 }
    00047         }
    00048         Block::Put(NULL, c)(Z0)(Z1);
    00049 }
    00050 #endif
    00051
    00052 #if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE || CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE
    00053 inline static void SSE2_Xor16(byte *a, const byte *b, const byte *c)
    00054 {
    00055 #if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE
    00056         *(__m128i *)a = _mm_xor_si128(*(__m128i *)b, *(__m128i *)c);
    00057 #else
    00058         asm ("movdqa %1, %%xmm0; pxor %2, %%xmm0; movdqa %%xmm0, %0;" : "=m" (a[0]) : "m"(b[0]), "m"(c[0]));
    00059 #endif
    00060 }
    00061 #endif
    00062
    00063 inline static void Xor16(byte *a, const byte *b, const byte *c)
    00064 {
    00065         ((word64 *)a)[0] = ((word64 *)b)[0] ^ ((word64 *)c)[0];
    00066         ((word64 *)a)[1] = ((word64 *)b)[1] ^ ((word64 *)c)[1];
    00067 }
    00068
    00069 void GCM_Base::SetKeyWithoutResync(const byte *userKey, size_t keylength, const NameValuePairs &params)
    00070 {
    00071         BlockCipher &blockCipher = AccessBlockCipher();
    00072         blockCipher.SetKey(userKey, keylength, params);
    00073
    00074         if (blockCipher.BlockSize() != REQUIRED_BLOCKSIZE)
    00075                 throw InvalidArgument(AlgorithmName() + ": block size of underlying block cipher is not 16");
    00076
    00077         int tableSize;
    00078         if (params.GetIntValue(Name::TableSize(), tableSize))
    00079                 tableSize = (tableSize >= 64*1024) ? 64*1024 : 2*1024;
    00080         else
    00081                 tableSize = (GetTablesOption() == GCM_64K_Tables) ? 64*1024 : 2*1024;
    00082
    00083 #if defined(_MSC_VER) && (_MSC_VER >= 1300 && _MSC_VER < 1400)
    00084         // VC 2003 workaround: compiler generates bad code for 64K tables
    00085         tableSize = 2*1024;
    00086 #endif
    00087
    00088         m_buffer.resize(3*REQUIRED_BLOCKSIZE + tableSize);
    00089         byte *hashKey = HashKey();
    00090         memset(hashKey, 0, REQUIRED_BLOCKSIZE);
    00091         blockCipher.ProcessBlock(hashKey);
    00092
    00093         byte *table = MulTable();
    00094         int i, j, k;
    00095         word64 V0, V1;
    00096
    00097         typedef BlockGetAndPut<word64, BigEndian> Block;
    00098         Block::Get(hashKey)(V0)(V1);
    00099
    00100         if (tableSize == 64*1024)
    00101         {
    00102                 for (i=0; i<128; i++)
    00103                 {
    00104                         k = i%8;
    00105                         Block::Put(NULL, table+(i/8)*256*16+(size_t(1)<<(11-k)))(V0)(V1);
    00106
    00107                         int x = (int)V1 & 1;
    00108                         V1 = (V1>>1) | (V0<<63);
    00109                         V0 = (V0>>1) ^ (x ? W64LIT(0xe1) << 56 : 0);
    00110                 }
    00111
    00112                 for (i=0; i<16; i++)
    00113                 {
    00114                         memset(table+i*256*16, 0, 16);
    00115 #if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE || CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE
    00116                         if (HasSSE2())
    00117                                 for (j=2; j<=0x80; j*=2)
    00118                                         for (k=1; k<j; k++)
    00119                                                 SSE2_Xor16(table+i*256*16+(j+k)*16, table+i*256*16+j*16, table+i*256*16+k*16);
    00120                         else
    00121 #endif
    00122                                 for (j=2; j<=0x80; j*=2)
    00123                                         for (k=1; k<j; k++)
    00124                                                 Xor16(table+i*256*16+(j+k)*16, table+i*256*16+j*16, table+i*256*16+k*16);
    00125                 }
    00126         }
    00127         else
    00128         {
    00129                 if (!s_reductionTableInitialized)
    00130                 {
    00131                         s_reductionTable[0] = 0;
    00132                         word16 x = 0x01c2;
    00133                         s_reductionTable[1] = ByteReverse(x);
    00134                         for (int i=2; i<=0x80; i*=2)
    00135                         {
    00136                                 x <<= 1;
    00137                                 s_reductionTable[i] = ByteReverse(x);
    00138                                 for (int j=1; j<i; j++)
    00139                                         s_reductionTable[i+j] = s_reductionTable[i] ^ s_reductionTable[j];
    00140                         }
    00141                         s_reductionTableInitialized = true;
    00142                 }
    00143
    00144                 for (i=0; i<128-24; i++)
    00145                 {
    00146                         k = i%32;
    00147                         if (k < 4)
    00148                                 Block::Put(NULL, table+1024+(i/32)*256+(size_t(1)<<(7-k)))(V0)(V1);
    00149                         else if (k < 8)
    00150                                 Block::Put(NULL, table+(i/32)*256+(size_t(1)<<(11-k)))(V0)(V1);
    00151
    00152                         int x = (int)V1 & 1;
    00153                         V1 = (V1>>1) | (V0<<63);
    00154                         V0 = (V0>>1) ^ (x ? W64LIT(0xe1) << 56 : 0);
    00155                 }
    00156
    00157                 for (i=0; i<4; i++)
    00158                 {
    00159                         memset(table+i*256, 0, 16);
    00160                         memset(table+1024+i*256, 0, 16);
    00161 #if CRYPTOPP_BOOL_SSE2_INTRINSICS_AVAILABLE || CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE
    00162                         if (HasSSE2())
    00163                                 for (j=2; j<=8; j*=2)
    00164                                         for (k=1; k<j; k++)
    00165                                         {
    00166                                                 SSE2_Xor16(table+i*256+(j+k)*16, table+i*256+j*16, table+i*256+k*16);
    00167                                                 SSE2_Xor16(table+1024+i*256+(j+k)*16, table+1024+i*256+j*16, table+1024+i*256+k*16);
    00168                                         }
    00169                         else
    00170 #endif
    00171                                 for (j=2; j<=8; j*=2)
    00172                                         for (k=1; k<j; k++)
    00173                                         {
    00174                                                 Xor16(table+i*256+(j+k)*16, table+i*256+j*16, table+i*256+k*16);
    00175                                                 Xor16(table+1024+i*256+(j+k)*16, table+1024+i*256+j*16, table+1024+i*256+k*16);
    00176                                         }
    00177                 }
    00178         }
    00179 }
    00180
    00181 void GCM_Base::Resync(const byte *iv, size_t len)
    00182 {
    00183         BlockCipher &cipher = AccessBlockCipher();
    00184         byte *hashBuffer = HashBuffer();
    00185
    00186         if (len == 12)
    00187         {
    00188                 memcpy(hashBuffer, iv, len);
    00189                 memset(hashBuffer+len, 0, 3);
    00190                 hashBuffer[len+3] = 1;
    00191         }
    00192         else
    00193         {
    00194                 size_t origLen = len;
    00195                 memset(hashBuffer, 0, HASH_BLOCKSIZE);
    00196
    00197                 if (len >= HASH_BLOCKSIZE)
    00198                 {
    00199                         len = GCM_Base::AuthenticateBlocks(iv, len);
    00200                         iv += (origLen - len);
    00201                 }
    00202
    00203                 if (len > 0)
    00204                 {
    00205                         memcpy(m_buffer, iv, len);
    00206                         memset(m_buffer+len, 0, HASH_BLOCKSIZE-len);
    00207                         GCM_Base::AuthenticateBlocks(m_buffer, HASH_BLOCKSIZE);
    00208                 }
    00209
    00210                 PutBlock<word64, BigEndian, true>(NULL, m_buffer)(0)(origLen*8);
    00211                 GCM_Base::AuthenticateBlocks(m_buffer, HASH_BLOCKSIZE);
    00212         }
    00213
    00214         if (m_state >= State_IVSet)
    00215                 m_ctr.Resynchronize(hashBuffer, REQUIRED_BLOCKSIZE);
    00216         else
    00217                 m_ctr.SetCipherWithIV(cipher, hashBuffer);
    00218
    00219         m_ctr.Seek(HASH_BLOCKSIZE);
    00220
    00221         memset(hashBuffer, 0, HASH_BLOCKSIZE);
    00222 }
    00223
    00224 unsigned int GCM_Base::OptimalDataAlignment() const
    00225 {
    00226         return HasSSE2() ? 16 : GetBlockCipher().OptimalDataAlignment();
    00227 }
    00228
    00229 #pragma warning(disable: 4731)  // frame pointer register 'ebp' modified by inline assembly code
    00230
    00231 #endif  // #ifndef CRYPTOPP_GENERATE_X64_MASM
    00232
    00233 #ifdef CRYPTOPP_X64_MASM_AVAILABLE
    00234 extern "C" {
    00235 void GCM_AuthenticateBlocks_2K(const byte *data, size_t blocks, word64 *hashBuffer, const word16 *reductionTable);
    00236 void GCM_AuthenticateBlocks_64K(const byte *data, size_t blocks, word64 *hashBuffer);
    00237 }
    00238 #endif
    00239
    00240 #ifndef CRYPTOPP_GENERATE_X64_MASM
    00241
    00242 size_t GCM_Base::AuthenticateBlocks(const byte *data, size_t len)
    00243 {
    00244         typedef BlockGetAndPut<word64, NativeByteOrder> Block;
    00245         word64 *hashBuffer = (word64 *)HashBuffer();
    00246
    00247         switch (2*(m_buffer.size()>=64*1024)
    00248 #if CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE || defined(CRYPTOPP_X64_MASM_AVAILABLE)
    00249                 + HasSSE2()
    00250 #endif
    00251                 )
    00252         {
    00253         case 0:         // non-SSE2 and 2K tables
    00254                 {
    00255                 byte *table = MulTable();
    00256                 word64 x0 = hashBuffer[0], x1 = hashBuffer[1];
    00257
    00258                 do
    00259                 {
    00260                         word64 y0, y1, a0, a1, b0, b1, c0, c1, d0, d1;
    00261                         Block::Get(data)(y0)(y1);
    00262                         x0 ^= y0;
    00263                         x1 ^= y1;
    00264
    00265                         data += HASH_BLOCKSIZE;
    00266                         len -= HASH_BLOCKSIZE;
    00267
    00268                         #define READ_TABLE_WORD64_COMMON(a, b, c, d)    *(word64 *)(table+(a*1024)+(b*256)+c+d*8)
    00269
    00270                         #ifdef IS_LITTLE_ENDIAN
    00271                                 #if CRYPTOPP_BOOL_SLOW_WORD64
    00272                                         word32 z0 = (word32)x0;
    00273                                         word32 z1 = (word32)(x0>>32);
    00274                                         word32 z2 = (word32)x1;
    00275                                         word32 z3 = (word32)(x1>>32);
    00276                                         #define READ_TABLE_WORD64(a, b, c, d, e)        READ_TABLE_WORD64_COMMON((d%2), c, (d?(z##c>>((d?d-1:0)*4))&0xf0:(z##c&0xf)<<4), e)
    00277                                 #else
    00278                                         #define READ_TABLE_WORD64(a, b, c, d, e)        READ_TABLE_WORD64_COMMON((d%2), c, ((d+8*b)?(x##a>>(((d+8*b)?(d+8*b)-1:1)*4))&0xf0:(x##a&0xf)<<4), e)
    00279                                 #endif
    00280                                 #define GF_MOST_SIG_8BITS(a) (a##1 >> 7*8)
    00281                                 #define GF_SHIFT_8(a) a##1 = (a##1 << 8) ^ (a##0 >> 7*8); a##0 <<= 8;
    00282                         #else
    00283                                 #define READ_TABLE_WORD64(a, b, c, d, e)        READ_TABLE_WORD64_COMMON((1-d%2), c, ((15-d-8*b)?(x##a>>(((15-d-8*b)?(15-d-8*b)-1:0)*4))&0xf0:(x##a&0xf)<<4), e)
    00284                                 #define GF_MOST_SIG_8BITS(a) (a##1 & 0xff)
    00285                                 #define GF_SHIFT_8(a) a##1 = (a##1 >> 8) ^ (a##0 << 7*8); a##0 >>= 8;
    00286                         #endif
    00287
    00288                         #define GF_MUL_32BY128(op, a, b, c)                                                                                     \
    00289                                 a0 op READ_TABLE_WORD64(a, b, c, 0, 0) ^ READ_TABLE_WORD64(a, b, c, 1, 0);\
    00290                                 a1 op READ_TABLE_WORD64(a, b, c, 0, 1) ^ READ_TABLE_WORD64(a, b, c, 1, 1);\
    00291                                 b0 op READ_TABLE_WORD64(a, b, c, 2, 0) ^ READ_TABLE_WORD64(a, b, c, 3, 0);\
    00292                                 b1 op READ_TABLE_WORD64(a, b, c, 2, 1) ^ READ_TABLE_WORD64(a, b, c, 3, 1);\
    00293                                 c0 op READ_TABLE_WORD64(a, b, c, 4, 0) ^ READ_TABLE_WORD64(a, b, c, 5, 0);\
    00294                                 c1 op READ_TABLE_WORD64(a, b, c, 4, 1) ^ READ_TABLE_WORD64(a, b, c, 5, 1);\
    00295                                 d0 op READ_TABLE_WORD64(a, b, c, 6, 0) ^ READ_TABLE_WORD64(a, b, c, 7, 0);\
    00296                                 d1 op READ_TABLE_WORD64(a, b, c, 6, 1) ^ READ_TABLE_WORD64(a, b, c, 7, 1);\
    00297
    00298                         GF_MUL_32BY128(=, 0, 0, 0)
    00299                         GF_MUL_32BY128(^=, 0, 1, 1)
    00300                         GF_MUL_32BY128(^=, 1, 0, 2)
    00301                         GF_MUL_32BY128(^=, 1, 1, 3)
    00302
    00303                         word32 r = (word32)s_reductionTable[GF_MOST_SIG_8BITS(d)] << 16;
    00304                         GF_SHIFT_8(d)
    00305                         c0 ^= d0; c1 ^= d1;
    00306                         r ^= (word32)s_reductionTable[GF_MOST_SIG_8BITS(c)] << 8;
    00307                         GF_SHIFT_8(c)
    00308                         b0 ^= c0; b1 ^= c1;
    00309                         r ^= s_reductionTable[GF_MOST_SIG_8BITS(b)];
    00310                         GF_SHIFT_8(b)
    00311                         a0 ^= b0; a1 ^= b1;
    00312                         a0 ^= ConditionalByteReverse<word64>(LITTLE_ENDIAN_ORDER, r);
    00313                         x0 = a0; x1 = a1;
    00314                 }
    00315                 while (len >= HASH_BLOCKSIZE);
    00316
    00317                 hashBuffer[0] = x0; hashBuffer[1] = x1;
    00318                 return len;
    00319                 }
    00320
    00321         case 2:         // non-SSE2 and 64K tables
    00322                 {
    00323                 byte *table = MulTable();
    00324                 word64 x0 = hashBuffer[0], x1 = hashBuffer[1];
    00325
    00326                 do
    00327                 {
    00328                         word64 y0, y1, a0, a1;
    00329                         Block::Get(data)(y0)(y1);
    00330                         x0 ^= y0;
    00331                         x1 ^= y1;
    00332
    00333                         data += HASH_BLOCKSIZE;
    00334                         len -= HASH_BLOCKSIZE;
    00335
    00336                         #undef READ_TABLE_WORD64_COMMON
    00337                         #undef READ_TABLE_WORD64
    00338
    00339                         #define READ_TABLE_WORD64_COMMON(a, c, d)       *(word64 *)(table+(a)*256*16+(c)+(d)*8)
    00340
    00341                         #ifdef IS_LITTLE_ENDIAN
    00342                                 #if CRYPTOPP_BOOL_SLOW_WORD64
    00343                                         word32 z0 = (word32)x0;
    00344                                         word32 z1 = (word32)(x0>>32);
    00345                                         word32 z2 = (word32)x1;
    00346                                         word32 z3 = (word32)(x1>>32);
    00347                                         #define READ_TABLE_WORD64(b, c, d, e)   READ_TABLE_WORD64_COMMON(c*4+d, (d?(z##c>>((d?d:1)*8-4))&0xff0:(z##c&0xff)<<4), e)
    00348                                 #else
    00349                                         #define READ_TABLE_WORD64(b, c, d, e)   READ_TABLE_WORD64_COMMON(c*4+d, ((d+4*(c%2))?(x##b>>(((d+4*(c%2))?(d+4*(c%2)):1)*8-4))&0xff0:(x##b&0xff)<<4), e)
    00350                                 #endif
    00351                         #else
    00352                                 #define READ_TABLE_WORD64(b, c, d, e)   READ_TABLE_WORD64_COMMON(c*4+d, ((7-d-4*(c%2))?(x##b>>(((7-d-4*(c%2))?(7-d-4*(c%2)):1)*8-4))&0xff0:(x##b&0xff)<<4), e)
    00353                         #endif
    00354
    00355                         #define GF_MUL_8BY128(op, b, c, d)              \
    00356                                 a0 op READ_TABLE_WORD64(b, c, d, 0);\
    00357                                 a1 op READ_TABLE_WORD64(b, c, d, 1);\
    00358
    00359                         GF_MUL_8BY128(=, 0, 0, 0)
    00360                         GF_MUL_8BY128(^=, 0, 0, 1)
    00361                         GF_MUL_8BY128(^=, 0, 0, 2)
    00362                         GF_MUL_8BY128(^=, 0, 0, 3)
    00363                         GF_MUL_8BY128(^=, 0, 1, 0)
    00364                         GF_MUL_8BY128(^=, 0, 1, 1)
    00365                         GF_MUL_8BY128(^=, 0, 1, 2)
    00366                         GF_MUL_8BY128(^=, 0, 1, 3)
    00367                         GF_MUL_8BY128(^=, 1, 2, 0)
    00368                         GF_MUL_8BY128(^=, 1, 2, 1)
    00369                         GF_MUL_8BY128(^=, 1, 2, 2)
    00370                         GF_MUL_8BY128(^=, 1, 2, 3)
    00371                         GF_MUL_8BY128(^=, 1, 3, 0)
    00372                         GF_MUL_8BY128(^=, 1, 3, 1)
    00373                         GF_MUL_8BY128(^=, 1, 3, 2)
    00374                         GF_MUL_8BY128(^=, 1, 3, 3)
    00375
    00376                         x0 = a0; x1 = a1;
    00377                 }
    00378                 while (len >= HASH_BLOCKSIZE);
    00379
    00380                 hashBuffer[0] = x0; hashBuffer[1] = x1;
    00381                 return len;
    00382                 }
    00383 #endif  // #ifndef CRYPTOPP_GENERATE_X64_MASM
    00384
    00385 #ifdef CRYPTOPP_X64_MASM_AVAILABLE
    00386         case 1:         // SSE2 and 2K tables
    00387                 GCM_AuthenticateBlocks_2K(data, len/16, hashBuffer, s_reductionTable);
    00388                 return len % 16;
    00389         case 3:         // SSE2 and 64K tables
    00390                 GCM_AuthenticateBlocks_64K(data, len/16, hashBuffer);
    00391                 return len % 16;
    00392 #endif
    00393
    00394 #if CRYPTOPP_BOOL_SSE2_ASM_AVAILABLE
    00395         case 1:         // SSE2 and 2K tables
    00396                 {
    00397                 #ifdef __GNUC__
    00398                         __asm__ __volatile__
    00399                         (
    00400                         ".intel_syntax noprefix;"
    00401                 #elif defined(CRYPTOPP_GENERATE_X64_MASM)
    00402                         ALIGN   8
    00403                         GCM_AuthenticateBlocks_2K       PROC FRAME
    00404                         rex_push_reg rsi
    00405                         push_reg rdi
    00406                         push_reg rbx
    00407                         .endprolog
    00408                         mov rsi, r8
    00409                         mov r11, r9
    00410                 #else
    00411                         AS2(    mov             WORD_REG(cx), data                      )
    00412                         AS2(    mov             WORD_REG(dx), len                       )
    00413                         AS2(    mov             WORD_REG(si), hashBuffer        )
    00414                         AS2(    shr             WORD_REG(dx), 4                         )
    00415                 #endif
    00416
    00417                 #if !defined(_MSC_VER) || (_MSC_VER < 1400)
    00418                         AS_PUSH_IF86(   bx)
    00419                 #endif
    00420                 AS_PUSH_IF86(   bp)
    00421
    00422                 #ifdef __GNUC__
    00423                         AS2(    mov             AS_REG_7, WORD_REG(di))
    00424                 #elif CRYPTOPP_BOOL_X86
    00425                         AS2(    lea             AS_REG_7, s_reductionTable)
    00426                 #endif
    00427
    00428                 AS2(    movdqa  xmm0, [WORD_REG(si)]                    )
    00429
    00430                 #define MUL_TABLE_0 WORD_REG(si) + 32
    00431                 #define MUL_TABLE_1 WORD_REG(si) + 32 + 1024
    00432                 #define RED_TABLE AS_REG_7
    00433
    00434                 ASL(0)
    00435                 AS2(    movdqu  xmm4, [WORD_REG(cx)]                    )
    00436                 AS2(    pxor    xmm0, xmm4                                              )
    00437
    00438                 AS2(    movd    ebx, xmm0                                               )
    00439                 AS2(    mov             eax, AS_HEX(f0f0f0f0)                   )
    00440                 AS2(    and             eax, ebx                                                )
    00441                 AS2(    shl             ebx, 4                                                  )
    00442                 AS2(    and             ebx, AS_HEX(f0f0f0f0)                   )
    00443                 AS2(    movzx   edi, ah                                                 )
    00444                 AS2(    movdqa  xmm5, XMMWORD_PTR [MUL_TABLE_1 + WORD_REG(di)]  )
    00445                 AS2(    movzx   edi, al                                 )
    00446                 AS2(    movdqa  xmm4, XMMWORD_PTR [MUL_TABLE_1 + WORD_REG(di)]  )
    00447                 AS2(    shr             eax, 16                                                 )
    00448                 AS2(    movzx   edi, ah                                 )
    00449                 AS2(    movdqa  xmm3, XMMWORD_PTR [MUL_TABLE_1 + WORD_REG(di)]  )
    00450                 AS2(    movzx   edi, al                                 )
    00451                 AS2(    movdqa  xmm2, XMMWORD_PTR [MUL_TABLE_1 + WORD_REG(di)]  )
    00452
    00453                 #define SSE2_MUL_32BITS(i)                                                                                      \
    00454                         AS2(    psrldq  xmm0, 4                                                                                 )\
    00455                         AS2(    movd    eax, xmm0                                                                               )\
    00456                         AS2(    and             eax, AS_HEX(f0f0f0f0)                                                                   )\
    00457                         AS2(    movzx   edi, bh                                                                                 )\
    00458                         AS2(    pxor    xmm5, XMMWORD_PTR [MUL_TABLE_0 + (i-1)*256 + WORD_REG(di)]      )\
    00459                         AS2(    movzx   edi, bl                                                                                 )\
    00460                         AS2(    pxor    xmm4, XMMWORD_PTR [MUL_TABLE_0 + (i-1)*256 + WORD_REG(di)]      )\
    00461                         AS2(    shr             ebx, 16                                                                                 )\
    00462                         AS2(    movzx   edi, bh                                                                                 )\
    00463                         AS2(    pxor    xmm3, XMMWORD_PTR [MUL_TABLE_0 + (i-1)*256 + WORD_REG(di)]      )\
    00464                         AS2(    movzx   edi, bl                                                                                 )\
    00465                         AS2(    pxor    xmm2, XMMWORD_PTR [MUL_TABLE_0 + (i-1)*256 + WORD_REG(di)]      )\
    00466                         AS2(    movd    ebx, xmm0                                                                               )\
    00467                         AS2(    shl             ebx, 4                                                                                  )\
    00468                         AS2(    and             ebx, AS_HEX(f0f0f0f0)                                                                   )\
    00469                         AS2(    movzx   edi, ah                                                                                 )\
    00470                         AS2(    pxor    xmm5, XMMWORD_PTR [MUL_TABLE_1 + i*256 + WORD_REG(di)]          )\
    00471                         AS2(    movzx   edi, al                                                                                 )\
    00472                         AS2(    pxor    xmm4, XMMWORD_PTR [MUL_TABLE_1 + i*256 + WORD_REG(di)]          )\
    00473                         AS2(    shr             eax, 16                                                                                 )\
    00474                         AS2(    movzx   edi, ah                                                                                 )\
    00475                         AS2(    pxor    xmm3, XMMWORD_PTR [MUL_TABLE_1 + i*256 + WORD_REG(di)]          )\
    00476                         AS2(    movzx   edi, al                                                                                 )\
    00477                         AS2(    pxor    xmm2, XMMWORD_PTR [MUL_TABLE_1 + i*256 + WORD_REG(di)]          )\
    00478
    00479                 SSE2_MUL_32BITS(1)
    00480                 SSE2_MUL_32BITS(2)
    00481                 SSE2_MUL_32BITS(3)
    00482
    00483                 AS2(    movzx   edi, bh                                 )
    00484                 AS2(    pxor    xmm5, XMMWORD_PTR [MUL_TABLE_0 + 3*256 + WORD_REG(di)]  )
    00485                 AS2(    movzx   edi, bl                                 )
    00486                 AS2(    pxor    xmm4, XMMWORD_PTR [MUL_TABLE_0 + 3*256 + WORD_REG(di)]  )
    00487                 AS2(    shr             ebx, 16                                         )
    00488                 AS2(    movzx   edi, bh                                 )
    00489                 AS2(    pxor    xmm3, XMMWORD_PTR [MUL_TABLE_0 + 3*256 + WORD_REG(di)]  )
    00490                 AS2(    movzx   edi, bl                                 )
    00491                 AS2(    pxor    xmm2, XMMWORD_PTR [MUL_TABLE_0 + 3*256 + WORD_REG(di)]  )
    00492
    00493                 AS2(    movdqa  xmm0, xmm3                                              )
    00494                 AS2(    pslldq  xmm3, 1                                                 )
    00495                 AS2(    pxor    xmm2, xmm3                                              )
    00496                 AS2(    movdqa  xmm1, xmm2                                              )
    00497                 AS2(    pslldq  xmm2, 1                                                 )
    00498                 AS2(    pxor    xmm5, xmm2                                              )
    00499
    00500                 AS2(    psrldq  xmm0, 15                                                )
    00501                 AS2(    movd    WORD_REG(di), xmm0                                      )
    00502                 AS2(    movzx   eax, WORD PTR [RED_TABLE + WORD_REG(di)*2]      )
    00503                 AS2(    shl             eax, 8                                                  )
    00504
    00505                 AS2(    movdqa  xmm0, xmm5                                              )
    00506                 AS2(    pslldq  xmm5, 1                                                 )
    00507                 AS2(    pxor    xmm4, xmm5                                              )
    00508
    00509                 AS2(    psrldq  xmm1, 15                                                )
    00510                 AS2(    movd    WORD_REG(di), xmm1                                      )
    00511                 AS2(    xor             ax, WORD PTR [RED_TABLE + WORD_REG(di)*2]       )
    00512                 AS2(    shl             eax, 8                                                  )
    00513
    00514                 AS2(    psrldq  xmm0, 15                                                )
    00515                 AS2(    movd    WORD_REG(di), xmm0                                      )
    00516                 AS2(    xor             ax, WORD PTR [RED_TABLE + WORD_REG(di)*2]       )
    00517
    00518                 AS2(    movd    xmm0, eax                                               )
    00519                 AS2(    pxor    xmm0, xmm4                                              )
    00520
    00521                 AS2(    add             WORD_REG(cx), 16                                        )
    00522                 AS2(    sub             WORD_REG(dx), 1                                         )
    00523                 ASJ(    jnz,    0, b                                                    )
    00524                 AS2(    movdqa  [WORD_REG(si)], xmm0                            )
    00525
    00526                 AS_POP_IF86(    bp)
    00527                 #if !defined(_MSC_VER) || (_MSC_VER < 1400)
    00528                         AS_POP_IF86(    bx)
    00529                 #endif
    00530
    00531                 #ifdef __GNUC__
    00532                                 ".att_syntax prefix;"
    00533                                         :
    00534                                         : "c" (data), "d" (len/16), "S" (hashBuffer), "D" (s_reductionTable)
    00535                                         : "memory", "cc", "%eax"
    00536                         #if CRYPTOPP_BOOL_X64
    00537                                         , "%ebx", "%r11"
    00538                         #endif
    00539                                 );
    00540                 #elif defined(CRYPTOPP_GENERATE_X64_MASM)
    00541                         pop rbx
    00542                         pop rdi
    00543                         pop rsi
    00544                         ret
    00545                         GCM_AuthenticateBlocks_2K ENDP
    00546                 #endif
    00547
    00548                 return len%16;
    00549                 }
    00550         case 3:         // SSE2 and 64K tables
    00551                 {
    00552                 #ifdef __GNUC__
    00553                         __asm__ __volatile__
    00554                         (
    00555                         ".intel_syntax noprefix;"
    00556                 #elif defined(CRYPTOPP_GENERATE_X64_MASM)
    00557                         ALIGN   8
    00558                         GCM_AuthenticateBlocks_64K      PROC FRAME
    00559                         rex_push_reg rsi
    00560                         push_reg rdi
    00561                         .endprolog
    00562                         mov rsi, r8
    00563                 #else
    00564                         AS2(    mov             WORD_REG(cx), data                      )
    00565                         AS2(    mov             WORD_REG(dx), len                       )
    00566                         AS2(    mov             WORD_REG(si), hashBuffer        )
    00567                         AS2(    shr             WORD_REG(dx), 4                         )
    00568                 #endif
    00569
    00570                 AS2(    movdqa  xmm0, [WORD_REG(si)]                            )
    00571
    00572                 #undef MUL_TABLE
    00573                 #define MUL_TABLE(i,j) WORD_REG(si) + 32 + (i*4+j)*256*16
    00574
    00575                 ASL(1)
    00576                 AS2(    movdqu  xmm1, [WORD_REG(cx)]                            )
    00577                 AS2(    pxor    xmm1, xmm0                                              )
    00578                 AS2(    pxor    xmm0, xmm0                                              )
    00579
    00580                 #undef SSE2_MUL_32BITS
    00581                 #define SSE2_MUL_32BITS(i)                                                              \
    00582                         AS2(    movd    eax, xmm1                                                       )\
    00583                         AS2(    psrldq  xmm1, 4                                                         )\
    00584                         AS2(    movzx   edi, al                                         )\
    00585                         AS2(    add             WORD_REG(di), WORD_REG(di)                                      )\
    00586                         AS2(    pxor    xmm0, [MUL_TABLE(i,0) + WORD_REG(di)*8] )\
    00587                         AS2(    movzx   edi, ah                                         )\
    00588                         AS2(    add             WORD_REG(di), WORD_REG(di)                                      )\
    00589                         AS2(    pxor    xmm0, [MUL_TABLE(i,1) + WORD_REG(di)*8] )\
    00590                         AS2(    shr             eax, 16                                                         )\
    00591                         AS2(    movzx   edi, al                                         )\
    00592                         AS2(    add             WORD_REG(di), WORD_REG(di)                                      )\
    00593                         AS2(    pxor    xmm0, [MUL_TABLE(i,2) + WORD_REG(di)*8] )\
    00594                         AS2(    movzx   edi, ah                                         )\
    00595                         AS2(    add             WORD_REG(di), WORD_REG(di)                                      )\
    00596                         AS2(    pxor    xmm0, [MUL_TABLE(i,3) + WORD_REG(di)*8] )\
    00597
    00598                 SSE2_MUL_32BITS(0)
    00599                 SSE2_MUL_32BITS(1)
    00600                 SSE2_MUL_32BITS(2)
    00601                 SSE2_MUL_32BITS(3)
    00602
    00603                 AS2(    add             WORD_REG(cx), 16                                        )
    00604                 AS2(    sub             WORD_REG(dx), 1                                         )
    00605                 ASJ(    jnz,    1, b                                                    )
    00606                 AS2(    movdqa  [WORD_REG(si)], xmm0                            )
    00607
    00608                 #ifdef __GNUC__
    00609                                 ".att_syntax prefix;"
    00610                                         :
    00611                                         : "c" (data), "d" (len/16), "S" (hashBuffer)
    00612                                         : "memory", "cc", "%edi", "%eax"
    00613                                 );
    00614                 #elif defined(CRYPTOPP_GENERATE_X64_MASM)
    00615                         pop rdi
    00616                         pop rsi
    00617                         ret
    00618                         GCM_AuthenticateBlocks_64K ENDP
    00619                 #endif
    00620
    00621                 return len%16;
    00622                 }
    00623 #endif
    00624 #ifndef CRYPTOPP_GENERATE_X64_MASM
    00625         }
    00626
    00627         return len%16;
    00628 }
    00629
    00630 void GCM_Base::AuthenticateLastHeaderBlock()
    00631 {
    00632         if (m_bufferedDataLength > 0)
    00633         {
    00634                 memset(m_buffer+m_bufferedDataLength, 0, HASH_BLOCKSIZE-m_bufferedDataLength);
    00635                 m_bufferedDataLength = 0;
    00636                 GCM_Base::AuthenticateBlocks(m_buffer, HASH_BLOCKSIZE);
    00637         }
    00638 }
    00639
    00640 void GCM_Base::AuthenticateLastConfidentialBlock()
    00641 {
    00642         GCM_Base::AuthenticateLastHeaderBlock();
    00643         PutBlock<word64, BigEndian, true>(NULL, m_buffer)(m_totalHeaderLength*8)(m_totalMessageLength*8);
    00644         GCM_Base::AuthenticateBlocks(m_buffer, HASH_BLOCKSIZE);
    00645 }
    00646
    00647 void GCM_Base::AuthenticateLastFooterBlock(byte *mac, size_t macSize)
    00648 {
    00649         m_ctr.Seek(0);
    00650         m_ctr.ProcessData(mac, HashBuffer(), macSize);
    00651 }
    00652
    00653 NAMESPACE_END
    00654
    00655 #endif  // #ifndef CRYPTOPP_GENERATE_X64_MASM
    00656 #endif