kmail

kmfolderindex.cpp

00001 /* -*- mode: C++; c-file-style: "gnu" -*-
00002     This file is part of KMail, the KDE mail client.
00003     Copyright (c) 2000 Don Sanders <sanders@kde.org>
00004 
00005     KMail is free software; you can redistribute it and/or modify it
00006     under the terms of the GNU General Public License, version 2, as
00007     published by the Free Software Foundation.
00008 
00009     KMail is distributed in the hope that it will be useful, but
00010     WITHOUT ANY WARRANTY; without even the implied warranty of
00011     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00012     General Public License for more details.
00013 
00014     You should have received a copy of the GNU General Public License
00015     along with this program; if not, write to the Free Software
00016     Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
00017 */
00018 
00019 #include "kmfolderindex.h"
00020 #include "kmfolder.h"
00021 #include "kmfoldertype.h"
00022 #include "kcursorsaver.h"
00023 #include <config.h>
00024 #include <tqfileinfo.h>
00025 #include <tqtimer.h>
00026 #include <kdebug.h>
00027 
00028 
00029 #define HAVE_MMAP //need to get this into autoconf FIXME  --Sam
00030 #include <unistd.h>
00031 #ifdef HAVE_MMAP
00032 #include <sys/mman.h>
00033 #endif
00034 
00035 // Current version of the table of contents (index) files
00036 #define INDEX_VERSION 1507
00037 
00038 #ifndef MAX_LINE
00039 #define MAX_LINE 4096
00040 #endif
00041 
00042 #ifndef INIT_MSGS
00043 #define INIT_MSGS 8
00044 #endif
00045 
00046 #include <errno.h>
00047 #include <assert.h>
00048 #include <utime.h>
00049 #include <fcntl.h>
00050 
00051 #ifdef HAVE_BYTESWAP_H
00052 #include <byteswap.h>
00053 #endif
00054 #include <kapplication.h>
00055 #include <kcursor.h>
00056 #include <kmessagebox.h>
00057 #include <klocale.h>
00058 #include "kmmsgdict.h"
00059 
00060 // We define functions as kmail_swap_NN so that we don't get compile errors
00061 // on platforms where bswap_NN happens to be a function instead of a define.
00062 
00063 /* Swap bytes in 32 bit value.  */
00064 #ifdef bswap_32
00065 #define kmail_swap_32(x) bswap_32(x)
00066 #else
00067 #define kmail_swap_32(x) \
00068      ((((x) & 0xff000000) >> 24) | (((x) & 0x00ff0000) >>  8) |           \
00069       (((x) & 0x0000ff00) <<  8) | (((x) & 0x000000ff) << 24))
00070 #endif
00071 
00072 #include <stdlib.h>
00073 #include <sys/types.h>
00074 #include <sys/stat.h>
00075 #include <sys/file.h>
00076 
00077 KMFolderIndex::KMFolderIndex(KMFolder* folder, const char* name)
00078   : FolderStorage(folder, name), mMsgList(INIT_MSGS)
00079 {
00080     mIndexStream = 0;
00081     mIndexStreamPtr = 0;
00082     mIndexStreamPtrLength = 0;
00083     mIndexSwapByteOrder = false;
00084     mIndexSizeOfLong = sizeof(long);
00085     mIndexId = 0;
00086     mHeaderOffset   = 0;
00087 }
00088 
00089 
00090 KMFolderIndex::~KMFolderIndex()
00091 {
00092 }
00093 
00094 
00095 TQString KMFolderIndex::indexLocation() const
00096 {
00097   TQString sLocation(folder()->path());
00098 
00099   if ( !sLocation.isEmpty() ) {
00100     sLocation += '/';
00101     sLocation += '.';
00102   }
00103   sLocation += dotEscape(fileName());
00104   sLocation += ".index";
00105 
00106   return sLocation;
00107 }
00108 
00109 int KMFolderIndex::updateIndex()
00110 {
00111   if (!mAutoCreateIndex)
00112     return 0;
00113   bool dirty = mDirty;
00114   mDirtyTimer->stop();
00115   for ( unsigned int i = 0; !dirty && i < mMsgList.high(); i++ ) {
00116     if ( mMsgList.at(i) ) {
00117       if ( !mMsgList.at(i)->syncIndexString() ) {
00118         dirty = true;
00119       }
00120     }
00121   }
00122   if (!dirty) { // Update successful
00123       touchFolderIdsFile();
00124       return 0;
00125   }
00126   return writeIndex();
00127 }
00128 
00129 int KMFolderIndex::writeIndex( bool createEmptyIndex )
00130 {
00131   TQString tempName;
00132   TQString indexName;
00133   mode_t old_umask;
00134   int len;
00135   const uchar *buffer = 0;
00136 
00137   indexName = indexLocation();
00138   tempName = indexName + ".temp";
00139   unlink(TQFile::encodeName(tempName));
00140 
00141   // We touch the folder, otherwise the index is regenerated, if KMail is
00142   // running, while the clock switches from daylight savings time to normal time
00143   utime(TQFile::encodeName(location()), 0);
00144 
00145   old_umask = umask(077);
00146   FILE *tmpIndexStream = fopen(TQFile::encodeName(tempName), "w");
00147   umask(old_umask);
00148   if (!tmpIndexStream)
00149     return errno;
00150 
00151   fprintf(tmpIndexStream, "# KMail-Index V%d\n", INDEX_VERSION);
00152 
00153   // Header
00154   Q_UINT32 byteOrder = 0x12345678;
00155   Q_UINT32 sizeOfLong = sizeof(long);
00156 
00157   Q_UINT32 header_length = sizeof(byteOrder)+sizeof(sizeOfLong);
00158   char pad_char = '\0';
00159   fwrite(&pad_char, sizeof(pad_char), 1, tmpIndexStream);
00160   fwrite(&header_length, sizeof(header_length), 1, tmpIndexStream);
00161 
00162   // Write header
00163   fwrite(&byteOrder, sizeof(byteOrder), 1, tmpIndexStream);
00164   fwrite(&sizeOfLong, sizeof(sizeOfLong), 1, tmpIndexStream);
00165 
00166   off_t nho = ftell(tmpIndexStream);
00167 
00168   if ( !createEmptyIndex ) {
00169     KMMsgBase* msgBase;
00170     for (unsigned int i=0; i<mMsgList.high(); i++)
00171     {
00172       if (!(msgBase = mMsgList.at(i))) continue;
00173       buffer = msgBase->asIndexString(len);
00174       fwrite(&len,sizeof(len), 1, tmpIndexStream);
00175 
00176       off_t tmp = ftell(tmpIndexStream);
00177       msgBase->setIndexOffset(tmp);
00178       msgBase->setIndexLength(len);
00179       if(fwrite(buffer, len, 1, tmpIndexStream) != 1)
00180     kdDebug(5006) << "Whoa! " << __FILE__ << ":" << __LINE__ << endl;
00181     }
00182   }
00183 
00184   int fError = ferror( tmpIndexStream );
00185   if( fError != 0 ) {
00186     fclose( tmpIndexStream );
00187     return fError;
00188   }
00189   if(    ( fflush( tmpIndexStream ) != 0 )
00190       || ( fsync( fileno( tmpIndexStream ) ) != 0 ) ) {
00191     int errNo = errno;
00192     fclose( tmpIndexStream );
00193     return errNo;
00194   }
00195   if( fclose( tmpIndexStream ) != 0 )
00196     return errno;
00197 
00198   ::rename(TQFile::encodeName(tempName), TQFile::encodeName(indexName));
00199   mHeaderOffset = nho;
00200   if (mIndexStream)
00201       fclose(mIndexStream);
00202 
00203   if ( createEmptyIndex )
00204     return 0;
00205 
00206   mIndexStream = fopen(TQFile::encodeName(indexName), "r+"); // index file
00207   assert( mIndexStream );
00208   fcntl(fileno(mIndexStream), F_SETFD, FD_CLOEXEC);
00209 
00210   updateIndexStreamPtr();
00211 
00212   writeFolderIdsFile();
00213 
00214   setDirty( false );
00215   return 0;
00216 }
00217 
00218 bool KMFolderIndex::readIndex()
00219 {
00220   if ( contentsType() != KMail::ContentsTypeMail ) {
00221     kdDebug(5006) << k_funcinfo << "Reading index for " << label() << endl;
00222   }
00223   Q_INT32 len;
00224   KMMsgInfo* mi;
00225 
00226   assert(mIndexStream != 0);
00227   rewind(mIndexStream);
00228 
00229   clearIndex();
00230   int version;
00231 
00232   setDirty( false );
00233 
00234   if (!readIndexHeader(&version)) return false;
00235   //kdDebug(5006) << "Index version for " << label() << " is " << version << endl;
00236 
00237   mUnreadMsgs = 0;
00238   mTotalMsgs = 0;
00239   mHeaderOffset = ftell(mIndexStream);
00240 
00241   clearIndex();
00242   while (!feof(mIndexStream))
00243   {
00244     mi = 0;
00245     if(version >= 1505) {
00246       if(!fread(&len, sizeof(len), 1, mIndexStream)) {
00247         // Seems to be normal?
00248         // kdDebug(5006) << k_funcinfo << " Unable to read length field!" << endl;
00249         break;
00250       }
00251 
00252       if (mIndexSwapByteOrder)
00253         len = kmail_swap_32(len);
00254 
00255       off_t offs = ftell(mIndexStream);
00256       if(fseek(mIndexStream, len, SEEK_CUR)) {
00257         kdDebug(5006) << k_funcinfo << " Unable to seek to the end of the message!" << endl;
00258         break;
00259       }
00260       mi = new KMMsgInfo(folder(), offs, len);
00261     }
00262     else
00263     {
00264       TQCString line(MAX_LINE);
00265       fgets(line.data(), MAX_LINE, mIndexStream);
00266       if (feof(mIndexStream)) break;
00267       if (*line.data() == '\0') {
00268         fclose(mIndexStream);
00269         mIndexStream = 0;
00270         clearIndex();
00271         return false;
00272       }
00273       mi = new KMMsgInfo(folder());
00274       mi->compat_fromOldIndexString(line, mConvertToUtf8);
00275     }
00276     if(!mi) {
00277       kdDebug(5006) << k_funcinfo << " Unable to create message info object!" << endl;
00278       break;
00279     }
00280 
00281     if (mi->isDeleted())
00282     {
00283       delete mi;  // skip messages that are marked as deleted
00284       setDirty( true );
00285       needsCompact = true;  //We have deleted messages - needs to be compacted
00286       continue;
00287     }
00288 #ifdef OBSOLETE
00289     else if (mi->isNew())
00290     {
00291       mi->setStatus(KMMsgStatusUnread);
00292       mi->setDirty(false);
00293     }
00294 #endif
00295     if ((mi->isNew()) || (mi->isUnread()) ||
00296         (folder() == kmkernel->outboxFolder()))
00297     {
00298       ++mUnreadMsgs;
00299       if (mUnreadMsgs == 0) ++mUnreadMsgs;
00300     }
00301     mMsgList.append(mi, false);
00302   }
00303   if( version < 1505)
00304   {
00305     mConvertToUtf8 = false;
00306     setDirty( true );
00307     writeIndex();
00308   }
00309 
00310   if ( version < 1507 ) {
00311     updateInvitationAndAddressFieldsFromContents();
00312     setDirty( true );
00313     writeIndex();
00314   }
00315 
00316   mTotalMsgs = mMsgList.count();
00317   if ( contentsType() != KMail::ContentsTypeMail ) {
00318     kdDebug(5006) << k_funcinfo << "Done reading the index for " << label() << ", we have " << mTotalMsgs << " messages." << endl;
00319   }
00320   return true;
00321 }
00322 
00323 
00324 int KMFolderIndex::count(bool cache) const
00325 {
00326   int res = FolderStorage::count(cache);
00327   if (res == -1)
00328     res = mMsgList.count();
00329   return res;
00330 }
00331 
00332 
00333 bool KMFolderIndex::readIndexHeader(int *gv)
00334 {
00335   int indexVersion;
00336   assert(mIndexStream != 0);
00337   mIndexSwapByteOrder = false;
00338   mIndexSizeOfLong = sizeof(long);
00339 
00340   int ret = fscanf(mIndexStream, "# KMail-Index V%d\n", &indexVersion);
00341   if ( ret == EOF || ret == 0 )
00342       return false; // index file has invalid header
00343   if(gv)
00344       *gv = indexVersion;
00345 
00346   // Check if the index is corrupted ("not compactable") and recreate it if necessary. See
00347   // FolderStorage::getMsg() for the detection code.
00348   if ( !mCompactable ) {
00349     kdWarning(5006) << "Index file " << indexLocation() << " is corrupted!!. Re-creating it." << endl;
00350     recreateIndex( false /* don't call readIndex() afterwards */ );
00351     return false;
00352   }
00353 
00354   if (indexVersion < 1505 ) {
00355       if(indexVersion == 1503) {
00356       kdDebug(5006) << "Converting old index file " << indexLocation() << " to utf-8" << endl;
00357       mConvertToUtf8 = true;
00358       }
00359       return true;
00360   } else if (indexVersion == 1505) {
00361   } else if (indexVersion < INDEX_VERSION && indexVersion != 1506) {
00362       kdDebug(5006) << "Index file " << indexLocation() << " is out of date. Re-creating it." << endl;
00363       createIndexFromContents();
00364       return false;
00365   } else if(indexVersion > INDEX_VERSION) {
00366       kapp->setOverrideCursor(KCursor::arrowCursor());
00367       int r = KMessageBox::questionYesNo(0,
00368                      i18n(
00369                         "The mail index for '%1' is from an unknown version of KMail (%2).\n"
00370                         "This index can be regenerated from your mail folder, but some "
00371                         "information, including status flags, may be lost. Do you wish "
00372                         "to downgrade your index file?") .arg(name()) .arg(indexVersion), TQString::null, i18n("Downgrade"), i18n("Do Not Downgrade") );
00373       kapp->restoreOverrideCursor();
00374       if (r == KMessageBox::Yes)
00375       createIndexFromContents();
00376       return false;
00377   }
00378   else {
00379       // Header
00380       Q_UINT32 byteOrder = 0;
00381       Q_UINT32 sizeOfLong = sizeof(long); // default
00382 
00383       Q_UINT32 header_length = 0;
00384       fseek(mIndexStream, sizeof(char), SEEK_CUR );
00385       fread(&header_length, sizeof(header_length), 1, mIndexStream);
00386       if (header_length > 0xFFFF)
00387          header_length = kmail_swap_32(header_length);
00388 
00389       off_t endOfHeader = ftell(mIndexStream) + header_length;
00390 
00391       bool needs_update = true;
00392       // Process available header parts
00393       if (header_length >= sizeof(byteOrder))
00394       {
00395          fread(&byteOrder, sizeof(byteOrder), 1, mIndexStream);
00396          mIndexSwapByteOrder = (byteOrder == 0x78563412);
00397          header_length -= sizeof(byteOrder);
00398 
00399          if (header_length >= sizeof(sizeOfLong))
00400          {
00401             fread(&sizeOfLong, sizeof(sizeOfLong), 1, mIndexStream);
00402             if (mIndexSwapByteOrder)
00403                sizeOfLong = kmail_swap_32(sizeOfLong);
00404             mIndexSizeOfLong = sizeOfLong;
00405             header_length -= sizeof(sizeOfLong);
00406             needs_update = false;
00407          }
00408       }
00409       if (needs_update || mIndexSwapByteOrder || (mIndexSizeOfLong != sizeof(long)))
00410     setDirty( true );
00411       // Seek to end of header
00412       fseek(mIndexStream, endOfHeader, SEEK_SET );
00413 
00414       if (mIndexSwapByteOrder)
00415          kdDebug(5006) << "Index File has byte order swapped!" << endl;
00416       if (mIndexSizeOfLong != sizeof(long))
00417          kdDebug(5006) << "Index File sizeOfLong is " << mIndexSizeOfLong << " while sizeof(long) is " << sizeof(long) << " !" << endl;
00418 
00419   }
00420   return true;
00421 }
00422 
00423 
00424 #ifdef HAVE_MMAP
00425 bool KMFolderIndex::updateIndexStreamPtr(bool just_close)
00426 #else
00427 bool KMFolderIndex::updateIndexStreamPtr(bool)
00428 #endif
00429 {
00430     // We touch the folder, otherwise the index is regenerated, if KMail is
00431     // running, while the clock switches from daylight savings time to normal time
00432     utime(TQFile::encodeName(location()), 0);
00433     utime(TQFile::encodeName(indexLocation()), 0);
00434     utime(TQFile::encodeName( KMMsgDict::getFolderIdsLocation( *this ) ), 0);
00435 
00436   mIndexSwapByteOrder = false;
00437 #ifdef HAVE_MMAP
00438     if(just_close) {
00439     if(mIndexStreamPtr)
00440         munmap((char *)mIndexStreamPtr, mIndexStreamPtrLength);
00441     mIndexStreamPtr = 0;
00442     mIndexStreamPtrLength = 0;
00443     return true;
00444     }
00445 
00446     assert(mIndexStream);
00447     struct stat stat_buf;
00448     if(fstat(fileno(mIndexStream), &stat_buf) == -1) {
00449     if(mIndexStreamPtr)
00450         munmap((char *)mIndexStreamPtr, mIndexStreamPtrLength);
00451     mIndexStreamPtr = 0;
00452     mIndexStreamPtrLength = 0;
00453     return false;
00454     }
00455     if(mIndexStreamPtr)
00456     munmap((char *)mIndexStreamPtr, mIndexStreamPtrLength);
00457     mIndexStreamPtrLength = stat_buf.st_size;
00458     mIndexStreamPtr = (uchar *)mmap(0, mIndexStreamPtrLength, PROT_READ, MAP_SHARED,
00459                     fileno(mIndexStream), 0);
00460     if(mIndexStreamPtr == MAP_FAILED) {
00461     mIndexStreamPtr = 0;
00462     mIndexStreamPtrLength = 0;
00463     return false;
00464     }
00465 #endif
00466     return true;
00467 }
00468 
00469 
00470 KMFolderIndex::IndexStatus KMFolderIndex::indexStatus()
00471 {
00472     if ( !mCompactable )
00473       return IndexCorrupt;
00474 
00475     TQFileInfo contInfo(location());
00476     TQFileInfo indInfo(indexLocation());
00477 
00478     if (!contInfo.exists()) return KMFolderIndex::IndexOk;
00479     if (!indInfo.exists()) return KMFolderIndex::IndexMissing;
00480 
00481     return ( contInfo.lastModified() > indInfo.lastModified() )
00482         ? KMFolderIndex::IndexTooOld
00483         : KMFolderIndex::IndexOk;
00484 }
00485 
00486 void KMFolderIndex::clearIndex(bool autoDelete, bool syncDict)
00487 {
00488     mMsgList.clear(autoDelete, syncDict);
00489 }
00490 
00491 
00492 void KMFolderIndex::truncateIndex()
00493 {
00494   if ( mHeaderOffset )
00495     truncate(TQFile::encodeName(indexLocation()), mHeaderOffset);
00496   else
00497     // The index file wasn't opened, so we don't know the header offset.
00498     // So let's just create a new empty index.
00499     writeIndex( true );
00500 }
00501 
00502 void KMFolderIndex::fillMessageDict()
00503 {
00504   open("fillDict");
00505   for (unsigned int idx = 0; idx < mMsgList.high(); idx++)
00506     if ( mMsgList.at( idx ) )
00507       KMMsgDict::mutableInstance()->insert(0, mMsgList.at( idx ), idx);
00508   close("fillDict");
00509 }
00510 
00511 
00512 KMMsgInfo* KMFolderIndex::setIndexEntry( int idx, KMMessage *msg )
00513 {
00514   KMMsgInfo *msgInfo = msg->msgInfo();
00515   if ( !msgInfo )
00516     msgInfo = new KMMsgInfo( folder() );
00517 
00518   *msgInfo = *msg;
00519   mMsgList.set( idx, msgInfo );
00520   msg->setMsgInfo( 0 );
00521   delete msg;
00522   return msgInfo;
00523 }
00524 
00525 void KMFolderIndex::recreateIndex( bool readIndexAfterwards )
00526 {
00527   kapp->setOverrideCursor(KCursor::arrowCursor());
00528   KMessageBox::information(0,
00529        i18n("The mail index for '%1' is corrupted and will be regenerated now, "
00530             "but some information, like status flags, might get lost.").arg(name()));
00531   kapp->restoreOverrideCursor();
00532   createIndexFromContents();
00533   if ( readIndexAfterwards ) {
00534     readIndex();
00535   }
00536 
00537   // Clear the corrupted flag
00538   mCompactable = true;
00539   writeConfig();
00540 }
00541 
00542 void KMFolderIndex::silentlyRecreateIndex()
00543 {
00544   Q_ASSERT( !isOpened() );
00545   open( "silentlyRecreateIndex" );
00546   KCursorSaver busy( KBusyPtr::busy() );
00547   createIndexFromContents();
00548   mCompactable = true;
00549   writeConfig();
00550   close( "silentlyRecreateIndex" );
00551 }
00552 
00553 void KMFolderIndex::updateInvitationAndAddressFieldsFromContents()
00554 {
00555   kdDebug(5006) << "Updating index for " << label() << ", this might take a while." << endl;
00556   for ( uint i = 0; i < mMsgList.size(); i++ ) {
00557     KMMsgInfo * const msgInfo = dynamic_cast<KMMsgInfo*>( mMsgList[i] );
00558     if ( msgInfo ) {
00559       DwString msgString( getDwString( i ) );
00560       if ( msgString.size() > 0 ) {
00561         KMMessage msg;
00562         msg.fromDwString( msgString, false );
00563         msg.updateInvitationState();
00564         if ( msg.status() & KMMsgStatusHasInvitation ) {
00565           msgInfo->setStatus( msgInfo->status() | KMMsgStatusHasInvitation );
00566         }
00567         if ( msg.status() & KMMsgStatusHasNoInvitation ) {
00568           msgInfo->setStatus( msgInfo->status() | KMMsgStatusHasNoInvitation );
00569         }
00570         msgInfo->setFrom( msg.from() );
00571         msgInfo->setTo( msg.to() );
00572       }
00573     }
00574   }
00575 }
00576 
00577 #include "kmfolderindex.moc"