filters

Decrypt.cc

00001 //========================================================================
00002 //
00003 // Decrypt.cc
00004 //
00005 // Copyright 1996-2002 Glyph & Cog, LLC
00006 //
00007 //========================================================================
00008 
00009 #include <aconf.h>
00010 
00011 #ifdef USE_GCC_PRAGMAS
00012 #pragma implementation
00013 #endif
00014 
00015 #include "gmem.h"
00016 #include "Decrypt.h"
00017 
00018 static void rc4InitKey(Guchar *key, int keyLen, Guchar *state);
00019 static Guchar rc4DecryptByte(Guchar *state, Guchar *x, Guchar *y, Guchar c);
00020 static void md5(Guchar *msg, int msgLen, Guchar *digest);
00021 
00022 static Guchar passwordPad[32] = {
00023   0x28, 0xbf, 0x4e, 0x5e, 0x4e, 0x75, 0x8a, 0x41,
00024   0x64, 0x00, 0x4e, 0x56, 0xff, 0xfa, 0x01, 0x08,
00025   0x2e, 0x2e, 0x00, 0xb6, 0xd0, 0x68, 0x3e, 0x80,
00026   0x2f, 0x0c, 0xa9, 0xfe, 0x64, 0x53, 0x69, 0x7a
00027 };
00028 
00029 //------------------------------------------------------------------------
00030 // Decrypt
00031 //------------------------------------------------------------------------
00032 
00033 Decrypt::Decrypt(Guchar *fileKey, int keyLength, int objNum, int objGen) {
00034   int i;
00035 
00036   // construct object key
00037   for (i = 0; i < keyLength; ++i) {
00038     objKey[i] = fileKey[i];
00039   }
00040   objKey[keyLength] = objNum & 0xff;
00041   objKey[keyLength + 1] = (objNum >> 8) & 0xff;
00042   objKey[keyLength + 2] = (objNum >> 16) & 0xff;
00043   objKey[keyLength + 3] = objGen & 0xff;
00044   objKey[keyLength + 4] = (objGen >> 8) & 0xff;
00045   md5(objKey, keyLength + 5, objKey);
00046 
00047   // set up for decryption
00048   x = y = 0;
00049   if ((objKeyLength = keyLength + 5) > 16) {
00050     objKeyLength = 16;
00051   }
00052   rc4InitKey(objKey, objKeyLength, state);
00053 }
00054 
00055 void Decrypt::reset() {
00056   x = y = 0;
00057   rc4InitKey(objKey, objKeyLength, state);
00058 }
00059 
00060 Guchar Decrypt::decryptByte(Guchar c) {
00061   return rc4DecryptByte(state, &x, &y, c);
00062 }
00063 
00064 GBool Decrypt::makeFileKey(int encVersion, int encRevision, int keyLength,
00065                GString *ownerKey, GString *userKey,
00066                int permissions, GString *fileID,
00067                GString *ownerPassword, GString *userPassword,
00068                Guchar *fileKey, GBool *ownerPasswordOk) {
00069   Guchar test[32], test2[32];
00070   GString *userPassword2;
00071   Guchar fState[256];
00072   Guchar tmpKey[16];
00073   Guchar fx, fy;
00074   int len, i, j;
00075 
00076   // try using the supplied owner password to generate the user password
00077   if (ownerPassword) {
00078     len = ownerPassword->getLength();
00079     if (len < 32) {
00080       memcpy(test, ownerPassword->getCString(), len);
00081       memcpy(test + len, passwordPad, 32 - len);
00082     } else {
00083       memcpy(test, ownerPassword->getCString(), 32);
00084     }
00085   } else {
00086     memcpy(test, passwordPad, 32);
00087   }
00088   md5(test, 32, test);
00089   if (encRevision == 3) {
00090     for (i = 0; i < 50; ++i) {
00091       md5(test, 16, test);
00092     }
00093   }
00094   if (encRevision == 2) {
00095     rc4InitKey(test, keyLength, fState);
00096     fx = fy = 0;
00097     for (i = 0; i < 32; ++i) {
00098       test2[i] = rc4DecryptByte(fState, &fx, &fy, ownerKey->getChar(i));
00099     }
00100   } else {
00101     memcpy(test2, ownerKey->getCString(), 32);
00102     for (i = 19; i >= 0; --i) {
00103       for (j = 0; j < keyLength; ++j) {
00104     tmpKey[j] = test[j] ^ i;
00105       }
00106       rc4InitKey(tmpKey, keyLength, fState);
00107       fx = fy = 0;
00108       for (j = 0; j < 32; ++j) {
00109     test2[j] = rc4DecryptByte(fState, &fx, &fy, test2[j]);
00110       }
00111     }
00112   }
00113   userPassword2 = new GString((char *)test2, 32);
00114   if (makeFileKey2(encVersion, encRevision, keyLength, ownerKey, userKey,
00115            permissions, fileID, userPassword2, fileKey)) {
00116     *ownerPasswordOk = gTrue;
00117     delete userPassword2;
00118     return gTrue;
00119   }
00120   *ownerPasswordOk = gFalse;
00121   delete userPassword2;
00122 
00123   // try using the supplied user password
00124   return makeFileKey2(encVersion, encRevision, keyLength, ownerKey, userKey,
00125               permissions, fileID, userPassword, fileKey);
00126 }
00127 
00128 GBool Decrypt::makeFileKey2(int /*encVersion*/, int encRevision, int keyLength,
00129                 GString *ownerKey, GString *userKey,
00130                 int permissions, GString *fileID,
00131                 GString *userPassword, Guchar *fileKey) {
00132   Guchar *buf;
00133   Guchar test[32];
00134   Guchar fState[256];
00135   Guchar tmpKey[16];
00136   Guchar fx, fy;
00137   int len, i, j;
00138   GBool ok;
00139 
00140   // generate file key
00141   buf = (Guchar *)gmalloc(68 + fileID->getLength());
00142   if (userPassword) {
00143     len = userPassword->getLength();
00144     if (len < 32) {
00145       memcpy(buf, userPassword->getCString(), len);
00146       memcpy(buf + len, passwordPad, 32 - len);
00147     } else {
00148       memcpy(buf, userPassword->getCString(), 32);
00149     }
00150   } else {
00151     memcpy(buf, passwordPad, 32);
00152   }
00153   memcpy(buf + 32, ownerKey->getCString(), 32);
00154   buf[64] = permissions & 0xff;
00155   buf[65] = (permissions >> 8) & 0xff;
00156   buf[66] = (permissions >> 16) & 0xff;
00157   buf[67] = (permissions >> 24) & 0xff;
00158   memcpy(buf + 68, fileID->getCString(), fileID->getLength());
00159   md5(buf, 68 + fileID->getLength(), fileKey);
00160   if (encRevision == 3) {
00161     for (i = 0; i < 50; ++i) {
00162       md5(fileKey, keyLength, fileKey);
00163     }
00164   }
00165 
00166   // test user password
00167   if (encRevision == 2) {
00168     rc4InitKey(fileKey, keyLength, fState);
00169     fx = fy = 0;
00170     for (i = 0; i < 32; ++i) {
00171       test[i] = rc4DecryptByte(fState, &fx, &fy, userKey->getChar(i));
00172     }
00173     ok = memcmp(test, passwordPad, 32) == 0;
00174   } else if (encRevision == 3) {
00175     memcpy(test, userKey->getCString(), 32);
00176     for (i = 19; i >= 0; --i) {
00177       for (j = 0; j < keyLength; ++j) {
00178     tmpKey[j] = fileKey[j] ^ i;
00179       }
00180       rc4InitKey(tmpKey, keyLength, fState);
00181       fx = fy = 0;
00182       for (j = 0; j < 32; ++j) {
00183     test[j] = rc4DecryptByte(fState, &fx, &fy, test[j]);
00184       }
00185     }
00186     memcpy(buf, passwordPad, 32);
00187     memcpy(buf + 32, fileID->getCString(), fileID->getLength());
00188     md5(buf, 32 + fileID->getLength(), buf);
00189     ok = memcmp(test, buf, 16) == 0;
00190   } else {
00191     ok = gFalse;
00192   }
00193 
00194   gfree(buf);
00195   return ok;
00196 }
00197 
00198 //------------------------------------------------------------------------
00199 // RC4-compatible decryption
00200 //------------------------------------------------------------------------
00201 
00202 static void rc4InitKey(Guchar *key, int keyLen, Guchar *state) {
00203   Guchar index1, index2;
00204   Guchar t;
00205   int i;
00206 
00207   for (i = 0; i < 256; ++i)
00208     state[i] = i;
00209   index1 = index2 = 0;
00210   for (i = 0; i < 256; ++i) {
00211     index2 = (key[index1] + state[i] + index2) % 256;
00212     t = state[i];
00213     state[i] = state[index2];
00214     state[index2] = t;
00215     index1 = (index1 + 1) % keyLen;
00216   }
00217 }
00218 
00219 static Guchar rc4DecryptByte(Guchar *state, Guchar *x, Guchar *y, Guchar c) {
00220   Guchar x1, y1, tx, ty;
00221 
00222   x1 = *x = (*x + 1) % 256;
00223   y1 = *y = (state[*x] + *y) % 256;
00224   tx = state[x1];
00225   ty = state[y1];
00226   state[x1] = ty;
00227   state[y1] = tx;
00228   return c ^ state[(tx + ty) % 256];
00229 }
00230 
00231 //------------------------------------------------------------------------
00232 // MD5 message digest
00233 //------------------------------------------------------------------------
00234 
00235 // this works around a bug in older Sun compilers
00236 static inline Gulong rotateLeft(Gulong x, int r) {
00237   x &= 0xffffffff;
00238   return ((x << r) | (x >> (32 - r))) & 0xffffffff;
00239 }
00240 
00241 static inline Gulong md5Round1(Gulong a, Gulong b, Gulong c, Gulong d,
00242                    Gulong Xk,  Gulong s, Gulong Ti) {
00243   return b + rotateLeft((a + ((b & c) | (~b & d)) + Xk + Ti), s);
00244 }
00245 
00246 static inline Gulong md5Round2(Gulong a, Gulong b, Gulong c, Gulong d,
00247                    Gulong Xk,  Gulong s, Gulong Ti) {
00248   return b + rotateLeft((a + ((b & d) | (c & ~d)) + Xk + Ti), s);
00249 }
00250 
00251 static inline Gulong md5Round3(Gulong a, Gulong b, Gulong c, Gulong d,
00252                    Gulong Xk,  Gulong s, Gulong Ti) {
00253   return b + rotateLeft((a + (b ^ c ^ d) + Xk + Ti), s);
00254 }
00255 
00256 static inline Gulong md5Round4(Gulong a, Gulong b, Gulong c, Gulong d,
00257                    Gulong Xk,  Gulong s, Gulong Ti) {
00258   return b + rotateLeft((a + (c ^ (b | ~d)) + Xk + Ti), s);
00259 }
00260 
00261 static void md5(Guchar *msg, int msgLen, Guchar *digest) {
00262   Gulong x[16];
00263   Gulong a, b, c, d, aa, bb, cc, dd;
00264   int n64;
00265   int i, j, k;
00266 
00267   // compute number of 64-byte blocks
00268   // (length + pad byte (0x80) + 8 bytes for length)
00269   n64 = (msgLen + 1 + 8 + 63) / 64;
00270 
00271   // initialize a, b, c, d
00272   a = 0x67452301;
00273   b = 0xefcdab89;
00274   c = 0x98badcfe;
00275   d = 0x10325476;
00276 
00277   // loop through blocks
00278   k = 0;
00279   for (i = 0; i < n64; ++i) {
00280 
00281     // grab a 64-byte block
00282     for (j = 0; j < 16 && k < msgLen - 3; ++j, k += 4)
00283       x[j] = (((((msg[k+3] << 8) + msg[k+2]) << 8) + msg[k+1]) << 8) + msg[k];
00284     if (i == n64 - 1) {
00285       if (k == msgLen - 3)
00286     x[j] = 0x80000000 + (((msg[k+2] << 8) + msg[k+1]) << 8) + msg[k];
00287       else if (k == msgLen - 2)
00288     x[j] = 0x800000 + (msg[k+1] << 8) + msg[k];
00289       else if (k == msgLen - 1)
00290     x[j] = 0x8000 + msg[k];
00291       else
00292     x[j] = 0x80;
00293       ++j;
00294       while (j < 16)
00295     x[j++] = 0;
00296       x[14] = msgLen << 3;
00297     }
00298 
00299     // save a, b, c, d
00300     aa = a;
00301     bb = b;
00302     cc = c;
00303     dd = d;
00304 
00305     // round 1
00306     a = md5Round1(a, b, c, d, x[0],   7, 0xd76aa478);
00307     d = md5Round1(d, a, b, c, x[1],  12, 0xe8c7b756);
00308     c = md5Round1(c, d, a, b, x[2],  17, 0x242070db);
00309     b = md5Round1(b, c, d, a, x[3],  22, 0xc1bdceee);
00310     a = md5Round1(a, b, c, d, x[4],   7, 0xf57c0faf);
00311     d = md5Round1(d, a, b, c, x[5],  12, 0x4787c62a);
00312     c = md5Round1(c, d, a, b, x[6],  17, 0xa8304613);
00313     b = md5Round1(b, c, d, a, x[7],  22, 0xfd469501);
00314     a = md5Round1(a, b, c, d, x[8],   7, 0x698098d8);
00315     d = md5Round1(d, a, b, c, x[9],  12, 0x8b44f7af);
00316     c = md5Round1(c, d, a, b, x[10], 17, 0xffff5bb1);
00317     b = md5Round1(b, c, d, a, x[11], 22, 0x895cd7be);
00318     a = md5Round1(a, b, c, d, x[12],  7, 0x6b901122);
00319     d = md5Round1(d, a, b, c, x[13], 12, 0xfd987193);
00320     c = md5Round1(c, d, a, b, x[14], 17, 0xa679438e);
00321     b = md5Round1(b, c, d, a, x[15], 22, 0x49b40821);
00322 
00323     // round 2
00324     a = md5Round2(a, b, c, d, x[1],   5, 0xf61e2562);
00325     d = md5Round2(d, a, b, c, x[6],   9, 0xc040b340);
00326     c = md5Round2(c, d, a, b, x[11], 14, 0x265e5a51);
00327     b = md5Round2(b, c, d, a, x[0],  20, 0xe9b6c7aa);
00328     a = md5Round2(a, b, c, d, x[5],   5, 0xd62f105d);
00329     d = md5Round2(d, a, b, c, x[10],  9, 0x02441453);
00330     c = md5Round2(c, d, a, b, x[15], 14, 0xd8a1e681);
00331     b = md5Round2(b, c, d, a, x[4],  20, 0xe7d3fbc8);
00332     a = md5Round2(a, b, c, d, x[9],   5, 0x21e1cde6);
00333     d = md5Round2(d, a, b, c, x[14],  9, 0xc33707d6);
00334     c = md5Round2(c, d, a, b, x[3],  14, 0xf4d50d87);
00335     b = md5Round2(b, c, d, a, x[8],  20, 0x455a14ed);
00336     a = md5Round2(a, b, c, d, x[13],  5, 0xa9e3e905);
00337     d = md5Round2(d, a, b, c, x[2],   9, 0xfcefa3f8);
00338     c = md5Round2(c, d, a, b, x[7],  14, 0x676f02d9);
00339     b = md5Round2(b, c, d, a, x[12], 20, 0x8d2a4c8a);
00340 
00341     // round 3
00342     a = md5Round3(a, b, c, d, x[5],   4, 0xfffa3942);
00343     d = md5Round3(d, a, b, c, x[8],  11, 0x8771f681);
00344     c = md5Round3(c, d, a, b, x[11], 16, 0x6d9d6122);
00345     b = md5Round3(b, c, d, a, x[14], 23, 0xfde5380c);
00346     a = md5Round3(a, b, c, d, x[1],   4, 0xa4beea44);
00347     d = md5Round3(d, a, b, c, x[4],  11, 0x4bdecfa9);
00348     c = md5Round3(c, d, a, b, x[7],  16, 0xf6bb4b60);
00349     b = md5Round3(b, c, d, a, x[10], 23, 0xbebfbc70);
00350     a = md5Round3(a, b, c, d, x[13],  4, 0x289b7ec6);
00351     d = md5Round3(d, a, b, c, x[0],  11, 0xeaa127fa);
00352     c = md5Round3(c, d, a, b, x[3],  16, 0xd4ef3085);
00353     b = md5Round3(b, c, d, a, x[6],  23, 0x04881d05);
00354     a = md5Round3(a, b, c, d, x[9],   4, 0xd9d4d039);
00355     d = md5Round3(d, a, b, c, x[12], 11, 0xe6db99e5);
00356     c = md5Round3(c, d, a, b, x[15], 16, 0x1fa27cf8);
00357     b = md5Round3(b, c, d, a, x[2],  23, 0xc4ac5665);
00358 
00359     // round 4
00360     a = md5Round4(a, b, c, d, x[0],   6, 0xf4292244);
00361     d = md5Round4(d, a, b, c, x[7],  10, 0x432aff97);
00362     c = md5Round4(c, d, a, b, x[14], 15, 0xab9423a7);
00363     b = md5Round4(b, c, d, a, x[5],  21, 0xfc93a039);
00364     a = md5Round4(a, b, c, d, x[12],  6, 0x655b59c3);
00365     d = md5Round4(d, a, b, c, x[3],  10, 0x8f0ccc92);
00366     c = md5Round4(c, d, a, b, x[10], 15, 0xffeff47d);
00367     b = md5Round4(b, c, d, a, x[1],  21, 0x85845dd1);
00368     a = md5Round4(a, b, c, d, x[8],   6, 0x6fa87e4f);
00369     d = md5Round4(d, a, b, c, x[15], 10, 0xfe2ce6e0);
00370     c = md5Round4(c, d, a, b, x[6],  15, 0xa3014314);
00371     b = md5Round4(b, c, d, a, x[13], 21, 0x4e0811a1);
00372     a = md5Round4(a, b, c, d, x[4],   6, 0xf7537e82);
00373     d = md5Round4(d, a, b, c, x[11], 10, 0xbd3af235);
00374     c = md5Round4(c, d, a, b, x[2],  15, 0x2ad7d2bb);
00375     b = md5Round4(b, c, d, a, x[9],  21, 0xeb86d391);
00376 
00377     // increment a, b, c, d
00378     a += aa;
00379     b += bb;
00380     c += cc;
00381     d += dd;
00382   }
00383 
00384   // break digest into bytes
00385   digest[0] = a & 0xff;
00386   digest[1] = (a >>= 8) & 0xff;
00387   digest[2] = (a >>= 8) & 0xff;
00388   digest[3] = (a >>= 8) & 0xff;
00389   digest[4] = b & 0xff;
00390   digest[5] = (b >>= 8) & 0xff;
00391   digest[6] = (b >>= 8) & 0xff;
00392   digest[7] = (b >>= 8) & 0xff;
00393   digest[8] = c & 0xff;
00394   digest[9] = (c >>= 8) & 0xff;
00395   digest[10] = (c >>= 8) & 0xff;
00396   digest[11] = (c >>= 8) & 0xff;
00397   digest[12] = d & 0xff;
00398   digest[13] = (d >>= 8) & 0xff;
00399   digest[14] = (d >>= 8) & 0xff;
00400   digest[15] = (d >>= 8) & 0xff;
00401 }
KDE Home | KDE Accessibility Home | Description of Access Keys