• Skip to content
  • Skip to link menu
  • KDE API Reference
  • kdelibs-4.14.38 API Reference
  • KDE Home
  • Contact Us
 

KIOSlave

  • kioslave
  • http
http_cache_cleaner.cpp
Go to the documentation of this file.
1/*
2This file is part of KDE
3
4 Copyright (C) 1999-2000 Waldo Bastian (bastian@kde.org)
5 Copyright (C) 2009 Andreas Hartmetz (ahartmetz@gmail.com)
6
7Permission is hereby granted, free of charge, to any person obtaining a copy
8of this software and associated documentation files (the "Software"), to deal
9in the Software without restriction, including without limitation the rights
10to use, copy, modify, merge, publish, distribute, and/or sell
11copies of the Software, and to permit persons to whom the Software is
12furnished to do so, subject to the following conditions:
13
14The above copyright notice and this permission notice shall be included in
15all copies or substantial portions of the Software.
16
17THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
20AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
21AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
22CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
23*/
24//----------------------------------------------------------------------------
25//
26// KDE HTTP Cache cleanup tool
27
28#include <cstring>
29#include <time.h>
30#include <stdlib.h>
31#include <zlib.h>
32
33#include <QtCore/QDir>
34#include <QtCore/QString>
35#include <QtCore/QTime>
36#include <QtDBus/QtDBus>
37#include <QtNetwork/QLocalServer>
38#include <QtNetwork/QLocalSocket>
39
40#include <kcmdlineargs.h>
41#include <kcomponentdata.h>
42#include <kdatetime.h>
43#include <kdebug.h>
44#include <kglobal.h>
45#include <klocale.h>
46#include <kprotocolmanager.h>
47#include <kstandarddirs.h>
48
49#include <unistd.h>
50
51time_t g_currentDate;
52int g_maxCacheAge;
53qint64 g_maxCacheSize;
54
55static const char appFullName[] = "org.kde.kio_http_cache_cleaner";
56static const char appName[] = "kio_http_cache_cleaner";
57
58// !START OF SYNC!
59// Keep the following in sync with the cache code in http.cpp
60
61static const int s_hashedUrlBits = 160; // this number should always be divisible by eight
62static const int s_hashedUrlNibbles = s_hashedUrlBits / 4;
63static const int s_hashedUrlBytes = s_hashedUrlBits / 8;
64
65static const char version[] = "A\n";
66
67// never instantiated, on-disk / wire format only
68struct SerializedCacheFileInfo {
69// from http.cpp
70 quint8 version[2];
71 quint8 compression; // for now fixed to 0
72 quint8 reserved; // for now; also alignment
73 static const int useCountOffset = 4;
74 qint32 useCount;
75 qint64 servedDate;
76 qint64 lastModifiedDate;
77 qint64 expireDate;
78 qint32 bytesCached;
79 static const int size = 36;
80
81 QString url;
82 QString etag;
83 QString mimeType;
84 QStringList responseHeaders; // including status response like "HTTP 200 OK"
85};
86
87static QString dateString(qint64 date)
88{
89 KDateTime dt;
90 dt.setTime_t(date);
91 return dt.toString(KDateTime::ISODate);
92}
93
94struct MiniCacheFileInfo {
95// data from cache entry file, or from scoreboard file
96 qint32 useCount;
97// from filesystem
98 qint64 lastUsedDate;
99 qint64 sizeOnDisk;
100 // we want to delete the least "useful" files and we'll have to sort a list for that...
101 bool operator<(const MiniCacheFileInfo &other) const;
102 void debugPrint() const
103 {
104 kDebug(7113) << "useCount:" << useCount
105 << "\nlastUsedDate:" << lastUsedDate
106 << "\nsizeOnDisk:" << sizeOnDisk << '\n';
107 }
108};
109
110struct CacheFileInfo : MiniCacheFileInfo {
111 quint8 version[2];
112 quint8 compression; // for now fixed to 0
113 quint8 reserved; // for now; also alignment
114
115
116 qint64 servedDate;
117 qint64 lastModifiedDate;
118 qint64 expireDate;
119 qint32 bytesCached;
120
121 QString baseName;
122 QString url;
123 QString etag;
124 QString mimeType;
125 QStringList responseHeaders; // including status response like "HTTP 200 OK"
126
127 void prettyPrint() const
128 {
129 QTextStream out(stdout, QIODevice::WriteOnly);
130 out << "File " << baseName << " version " << version[0] << version[1];
131 out << "\n cached bytes " << bytesCached << " useCount " << useCount;
132 out << "\n servedDate " << dateString(servedDate);
133 out << "\n lastModifiedDate " << dateString(lastModifiedDate);
134 out << "\n expireDate " << dateString(expireDate);
135 out << "\n entity tag " << etag;
136 out << "\n encoded URL " << url;
137 out << "\n mimetype " << mimeType;
138 out << "\nResponse headers follow...\n";
139 Q_FOREACH (const QString &h, responseHeaders) {
140 out << h << '\n';
141 }
142 }
143};
144
145
146bool MiniCacheFileInfo::operator<(const MiniCacheFileInfo &other) const
147{
148 const int thisUseful = useCount / qMax(g_currentDate - lastUsedDate, qint64(1));
149 const int otherUseful = other.useCount / qMax(g_currentDate - other.lastUsedDate, qint64(1));
150 return thisUseful < otherUseful;
151}
152
153bool CacheFileInfoPtrLessThan(const CacheFileInfo *cf1, const CacheFileInfo *cf2)
154{
155 return *cf1 < *cf2;
156}
157
158enum OperationMode {
159 CleanCache = 0,
160 DeleteCache,
161 FileInfo
162};
163
164static bool timeSizeFits(qint64 intTime)
165{
166 time_t tTime = static_cast<time_t>(intTime);
167 qint64 check = static_cast<qint64>(tTime);
168 return check == intTime;
169}
170
171static bool readBinaryHeader(const QByteArray &d, CacheFileInfo *fi)
172{
173 if (d.size() < SerializedCacheFileInfo::size) {
174 kDebug(7113) << "readBinaryHeader(): file too small?";
175 return false;
176 }
177 QDataStream stream(d);
178 stream.setVersion(QDataStream::Qt_4_5);
179
180 stream >> fi->version[0];
181 stream >> fi->version[1];
182 if (fi->version[0] != version[0] || fi->version[1] != version[1]) {
183 kDebug(7113) << "readBinaryHeader(): wrong magic bytes";
184 return false;
185 }
186 stream >> fi->compression;
187 stream >> fi->reserved;
188
189 stream >> fi->useCount;
190
191 stream >> fi->servedDate;
192 stream >> fi->lastModifiedDate;
193 stream >> fi->expireDate;
194 bool timeSizeOk = timeSizeFits(fi->servedDate) && timeSizeFits(fi->lastModifiedDate) &&
195 timeSizeFits(fi->expireDate);
196
197 stream >> fi->bytesCached;
198 return timeSizeOk;
199}
200
201static QString filenameFromUrl(const QByteArray &url)
202{
203 QCryptographicHash hash(QCryptographicHash::Sha1);
204 hash.addData(url);
205 return QString::fromLatin1(hash.result().toHex());
206}
207
208static QString filePath(const QString &baseName)
209{
210 QString cacheDirName = KGlobal::dirs()->saveLocation("cache", "http");
211 if (!cacheDirName.endsWith('/')) {
212 cacheDirName.append('/');
213 }
214 return cacheDirName + baseName;
215}
216
217static bool readLineChecked(QIODevice *dev, QByteArray *line)
218{
219 *line = dev->readLine(8192);
220 // if nothing read or the line didn't fit into 8192 bytes(!)
221 if (line->isEmpty() || !line->endsWith('\n')) {
222 return false;
223 }
224 // we don't actually want the newline!
225 line->chop(1);
226 return true;
227}
228
229static bool readTextHeader(QFile *file, CacheFileInfo *fi, OperationMode mode)
230{
231 bool ok = true;
232 QByteArray readBuf;
233
234 ok = ok && readLineChecked(file, &readBuf);
235 fi->url = QString::fromLatin1(readBuf);
236 if (filenameFromUrl(readBuf) != QFileInfo(*file).baseName()) {
237 kDebug(7103) << "You have witnessed a very improbable hash collision!";
238 return false;
239 }
240
241 // only read the necessary info for cache cleaning. Saves time and (more importantly) memory.
242 if (mode != FileInfo) {
243 return true;
244 }
245
246 ok = ok && readLineChecked(file, &readBuf);
247 fi->etag = QString::fromLatin1(readBuf);
248
249 ok = ok && readLineChecked(file, &readBuf);
250 fi->mimeType = QString::fromLatin1(readBuf);
251
252 // read as long as no error and no empty line found
253 while (true) {
254 ok = ok && readLineChecked(file, &readBuf);
255 if (ok && !readBuf.isEmpty()) {
256 fi->responseHeaders.append(QString::fromLatin1(readBuf));
257 } else {
258 break;
259 }
260 }
261 return ok; // it may still be false ;)
262}
263
264// TODO common include file with http.cpp?
265enum CacheCleanerCommand {
266 InvalidCommand = 0,
267 CreateFileNotificationCommand,
268 UpdateFileCommand
269};
270
271static bool readCacheFile(const QString &baseName, CacheFileInfo *fi, OperationMode mode)
272{
273 QFile file(filePath(baseName));
274 if (!file.open(QIODevice::ReadOnly)) {
275 return false;
276 }
277 fi->baseName = baseName;
278
279 QByteArray header = file.read(SerializedCacheFileInfo::size);
280 // do *not* modify/delete the file if we're in file info mode.
281 if (!(readBinaryHeader(header, fi) && readTextHeader(&file, fi, mode)) && mode != FileInfo) {
282 kDebug(7113) << "read(Text|Binary)Header() returned false, deleting file" << baseName;
283 file.remove();
284 return false;
285 }
286 // get meta-information from the filesystem
287 QFileInfo fileInfo(file);
288 fi->lastUsedDate = fileInfo.lastModified().toTime_t();
289 fi->sizeOnDisk = fileInfo.size();
290 return true;
291}
292
293class Scoreboard;
294
295class CacheIndex
296{
297public:
298 explicit CacheIndex(const QString &baseName)
299 {
300 QByteArray ba = baseName.toLatin1();
301 const int sz = ba.size();
302 const char *input = ba.constData();
303 Q_ASSERT(sz == s_hashedUrlNibbles);
304
305 int translated = 0;
306 for (int i = 0; i < sz; i++) {
307 int c = input[i];
308
309 if (c >= '0' && c <= '9') {
310 translated |= c - '0';
311 } else if (c >= 'a' && c <= 'f') {
312 translated |= c - 'a' + 10;
313 } else {
314 Q_ASSERT(false);
315 }
316
317 if (i & 1) {
318 // odd index
319 m_index[i >> 1] = translated;
320 translated = 0;
321 } else {
322 translated = translated << 4;
323 }
324 }
325
326 computeHash();
327 }
328
329 bool operator==(const CacheIndex &other) const
330 {
331 const bool isEqual = memcmp(m_index, other.m_index, s_hashedUrlBytes) == 0;
332 if (isEqual) {
333 Q_ASSERT(m_hash == other.m_hash);
334 }
335 return isEqual;
336 }
337
338private:
339 explicit CacheIndex(const QByteArray &index)
340 {
341 Q_ASSERT(index.length() >= s_hashedUrlBytes);
342 memcpy(m_index, index.constData(), s_hashedUrlBytes);
343 computeHash();
344 }
345
346 void computeHash()
347 {
348 uint hash = 0;
349 const int ints = s_hashedUrlBytes / sizeof(uint);
350 for (int i = 0; i < ints; i++) {
351 hash ^= reinterpret_cast<uint *>(&m_index[0])[i];
352 }
353 if (const int bytesLeft = s_hashedUrlBytes % sizeof(uint)) {
354 // dead code until a new url hash algorithm or architecture with sizeof(uint) != 4 appears.
355 // we have the luxury of ignoring endianness because the hash is never written to disk.
356 // just merge the bits into the the hash in some way.
357 const int offset = ints * sizeof(uint);
358 for (int i = 0; i < bytesLeft; i++) {
359 hash ^= static_cast<uint>(m_index[offset + i]) << (i * 8);
360 }
361 }
362 m_hash = hash;
363 }
364
365 friend uint qHash(const CacheIndex &);
366 friend class Scoreboard;
367
368 quint8 m_index[s_hashedUrlBytes]; // packed binary version of the hexadecimal name
369 uint m_hash;
370};
371
372uint qHash(const CacheIndex &ci)
373{
374 return ci.m_hash;
375}
376
377
378static CacheCleanerCommand readCommand(const QByteArray &cmd, CacheFileInfo *fi)
379{
380 readBinaryHeader(cmd, fi);
381 QDataStream stream(cmd);
382 stream.skipRawData(SerializedCacheFileInfo::size);
383
384 quint32 ret;
385 stream >> ret;
386
387 QByteArray baseName;
388 baseName.resize(s_hashedUrlNibbles);
389 stream.readRawData(baseName.data(), s_hashedUrlNibbles);
390 Q_ASSERT(stream.atEnd());
391 fi->baseName = QString::fromLatin1(baseName);
392
393 Q_ASSERT(ret == CreateFileNotificationCommand || ret == UpdateFileCommand);
394 return static_cast<CacheCleanerCommand>(ret);
395}
396
397
398// never istantiated, on-disk format only
399struct ScoreboardEntry {
400// from scoreboard file
401 quint8 index[s_hashedUrlBytes];
402 static const int indexSize = s_hashedUrlBytes;
403 qint32 useCount;
404// from scoreboard file, but compared with filesystem to see if scoreboard has current data
405 qint64 lastUsedDate;
406 qint32 sizeOnDisk;
407 static const int size = 36;
408 // we want to delete the least "useful" files and we'll have to sort a list for that...
409 bool operator<(const MiniCacheFileInfo &other) const;
410};
411
412
413class Scoreboard
414{
415public:
416 Scoreboard()
417 {
418 // read in the scoreboard...
419 QFile sboard(filePath(QLatin1String("scoreboard")));
420 sboard.open(QIODevice::ReadOnly);
421 while (true) {
422 QByteArray baIndex = sboard.read(ScoreboardEntry::indexSize);
423 QByteArray baRest = sboard.read(ScoreboardEntry::size - ScoreboardEntry::indexSize);
424 if (baIndex.size() + baRest.size() != ScoreboardEntry::size) {
425 break;
426 }
427
428 const QString entryBasename = QString::fromLatin1(baIndex.toHex());
429 MiniCacheFileInfo mcfi;
430 if (readAndValidateMcfi(baRest, entryBasename, &mcfi)) {
431 m_scoreboard.insert(CacheIndex(baIndex), mcfi);
432 }
433 }
434 }
435
436 void writeOut()
437 {
438 // write out the scoreboard
439 QFile sboard(filePath(QLatin1String("scoreboard")));
440 if (!sboard.open(QIODevice::WriteOnly | QIODevice::Truncate)) {
441 return;
442 }
443 QDataStream stream(&sboard);
444
445 QHash<CacheIndex, MiniCacheFileInfo>::ConstIterator it = m_scoreboard.constBegin();
446 for (; it != m_scoreboard.constEnd(); ++it) {
447 const char *indexData = reinterpret_cast<const char *>(it.key().m_index);
448 stream.writeRawData(indexData, s_hashedUrlBytes);
449
450 stream << it.value().useCount;
451 stream << it.value().lastUsedDate;
452 stream << it.value().sizeOnDisk;
453 }
454 }
455
456 bool fillInfo(const QString &baseName, MiniCacheFileInfo *mcfi)
457 {
458 QHash<CacheIndex, MiniCacheFileInfo>::ConstIterator it =
459 m_scoreboard.constFind(CacheIndex(baseName));
460 if (it == m_scoreboard.constEnd()) {
461 return false;
462 }
463 *mcfi = it.value();
464 return true;
465 }
466
467 qint64 runCommand(const QByteArray &cmd)
468 {
469 // execute the command; return number of bytes if a new file was created, zero otherwise.
470 Q_ASSERT(cmd.size() == 80);
471 CacheFileInfo fi;
472 const CacheCleanerCommand ccc = readCommand(cmd, &fi);
473 QString fileName = filePath(fi.baseName);
474
475 switch (ccc) {
476 case CreateFileNotificationCommand:
477 kDebug(7113) << "CreateNotificationCommand for" << fi.baseName;
478 if (!readBinaryHeader(cmd, &fi)) {
479 return 0;
480 }
481 break;
482
483 case UpdateFileCommand: {
484 kDebug(7113) << "UpdateFileCommand for" << fi.baseName;
485 QFile file(fileName);
486 file.open(QIODevice::ReadWrite);
487
488 CacheFileInfo fiFromDisk;
489 QByteArray header = file.read(SerializedCacheFileInfo::size);
490 if (!readBinaryHeader(header, &fiFromDisk) || fiFromDisk.bytesCached != fi.bytesCached) {
491 return 0;
492 }
493
494 // adjust the use count, to make sure that we actually count up. (slaves read the file
495 // asynchronously...)
496 const quint32 newUseCount = fiFromDisk.useCount + 1;
497 QByteArray newHeader = cmd.mid(0, SerializedCacheFileInfo::size);
498 {
499 QDataStream stream(&newHeader, QIODevice::WriteOnly);
500 stream.skipRawData(SerializedCacheFileInfo::useCountOffset);
501 stream << newUseCount;
502 }
503
504 file.seek(0);
505 file.write(newHeader);
506 file.close();
507
508 if (!readBinaryHeader(newHeader, &fi)) {
509 return 0;
510 }
511 break;
512 }
513
514 default:
515 kDebug(7113) << "received invalid command";
516 return 0;
517 }
518
519 QFileInfo fileInfo(fileName);
520 fi.lastUsedDate = fileInfo.lastModified().toTime_t();
521 fi.sizeOnDisk = fileInfo.size();
522 fi.debugPrint();
523 // a CacheFileInfo is-a MiniCacheFileInfo which enables the following assignment...
524 add(fi);
525 // finally, return cache dir growth (only relevant if a file was actually created!)
526 return ccc == CreateFileNotificationCommand ? fi.sizeOnDisk : 0;
527 }
528
529 void add(const CacheFileInfo &fi)
530 {
531 m_scoreboard[CacheIndex(fi.baseName)] = fi;
532 }
533
534 void remove(const QString &basename)
535 {
536 m_scoreboard.remove(CacheIndex(basename));
537 }
538
539 // keep memory usage reasonably low - otherwise entries of nonexistent files don't hurt.
540 void maybeRemoveStaleEntries(const QList<CacheFileInfo *> &fiList)
541 {
542 // don't bother when there are a few bogus entries
543 if (m_scoreboard.count() < fiList.count() + 100) {
544 return;
545 }
546 kDebug(7113) << "we have too many fake/stale entries, cleaning up...";
547 QSet<CacheIndex> realFiles;
548 Q_FOREACH (CacheFileInfo *fi, fiList) {
549 realFiles.insert(CacheIndex(fi->baseName));
550 }
551 QHash<CacheIndex, MiniCacheFileInfo>::Iterator it = m_scoreboard.begin();
552 while (it != m_scoreboard.end()) {
553 if (realFiles.contains(it.key())) {
554 ++it;
555 } else {
556 it = m_scoreboard.erase(it);
557 }
558 }
559 }
560
561private:
562 bool readAndValidateMcfi(const QByteArray &rawData, const QString &basename, MiniCacheFileInfo *mcfi)
563 {
564 QDataStream stream(rawData);
565 stream >> mcfi->useCount;
566 // check those against filesystem
567 stream >> mcfi->lastUsedDate;
568 stream >> mcfi->sizeOnDisk;
569
570 QFileInfo fileInfo(filePath(basename));
571 if (!fileInfo.exists()) {
572 return false;
573 }
574 bool ok = true;
575 ok = ok && fileInfo.lastModified().toTime_t() == mcfi->lastUsedDate;
576 ok = ok && fileInfo.size() == mcfi->sizeOnDisk;
577 if (!ok) {
578 // size or last-modified date not consistent with entry file; reload useCount
579 // note that avoiding to open the file is the whole purpose of the scoreboard - we only
580 // open the file if we really have to.
581 QFile entryFile(fileInfo.absoluteFilePath());
582 if (!entryFile.open(QIODevice::ReadOnly)) {
583 return false;
584 }
585 if (entryFile.size() < SerializedCacheFileInfo::size) {
586 return false;
587 }
588 QDataStream stream(&entryFile);
589 stream.skipRawData(SerializedCacheFileInfo::useCountOffset);
590
591 stream >> mcfi->useCount;
592 mcfi->lastUsedDate = fileInfo.lastModified().toTime_t();
593 mcfi->sizeOnDisk = fileInfo.size();
594 ok = true;
595 }
596 return ok;
597 }
598
599 QHash<CacheIndex, MiniCacheFileInfo> m_scoreboard;
600};
601
602
603// Keep the above in sync with the cache code in http.cpp
604// !END OF SYNC!
605
606// remove files and directories used by earlier versions of the HTTP cache.
607static void removeOldFiles()
608{
609 const char *oldDirs = "0abcdefghijklmnopqrstuvwxyz";
610 const int n = strlen(oldDirs);
611 QDir cacheRootDir(filePath(QString()));
612 for (int i = 0; i < n; i++) {
613 QString dirName = QString::fromLatin1(&oldDirs[i], 1);
614 // delete files in directory...
615 Q_FOREACH (const QString &baseName, QDir(filePath(dirName)).entryList()) {
616 QFile::remove(filePath(dirName + '/' + baseName));
617 }
618 // delete the (now hopefully empty!) directory itself
619 cacheRootDir.rmdir(dirName);
620 }
621 QFile::remove(filePath(QLatin1String("cleaned")));
622}
623
624class CacheCleaner
625{
626public:
627 CacheCleaner(const QDir &cacheDir)
628 : m_totalSizeOnDisk(0)
629 {
630 kDebug(7113);
631 m_fileNameList = cacheDir.entryList();
632 }
633
634 // Delete some of the files that need to be deleted. Return true when done, false otherwise.
635 // This makes interleaved cleaning / serving ioslaves possible.
636 bool processSlice(Scoreboard *scoreboard = 0)
637 {
638 QTime t;
639 t.start();
640 // phase one: gather information about cache files
641 if (!m_fileNameList.isEmpty()) {
642 while (t.elapsed() < 100 && !m_fileNameList.isEmpty()) {
643 QString baseName = m_fileNameList.takeFirst();
644 // check if the filename is of the $s_hashedUrlNibbles letters, 0...f type
645 if (baseName.length() < s_hashedUrlNibbles) {
646 continue;
647 }
648 bool nameOk = true;
649 for (int i = 0; i < s_hashedUrlNibbles && nameOk; i++) {
650 QChar c = baseName[i];
651 nameOk = (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f');
652 }
653 if (!nameOk) {
654 continue;
655 }
656 if (baseName.length() > s_hashedUrlNibbles) {
657 if (g_currentDate - QFileInfo(filePath(baseName)).lastModified().toTime_t() > 15*60) {
658 // it looks like a temporary file that hasn't been touched in > 15 minutes...
659 QFile::remove(filePath(baseName));
660 }
661 // the temporary file might still be written to, leave it alone
662 continue;
663 }
664
665 CacheFileInfo *fi = new CacheFileInfo();
666 fi->baseName = baseName;
667
668 bool gotInfo = false;
669 if (scoreboard) {
670 gotInfo = scoreboard->fillInfo(baseName, fi);
671 }
672 if (!gotInfo) {
673 gotInfo = readCacheFile(baseName, fi, CleanCache);
674 if (gotInfo && scoreboard) {
675 scoreboard->add(*fi);
676 }
677 }
678 if (gotInfo) {
679 m_fiList.append(fi);
680 m_totalSizeOnDisk += fi->sizeOnDisk;
681 } else {
682 delete fi;
683 }
684 }
685 kDebug(7113) << "total size of cache files is" << m_totalSizeOnDisk;
686
687 if (m_fileNameList.isEmpty()) {
688 // final step of phase one
689 qSort(m_fiList.begin(), m_fiList.end(), CacheFileInfoPtrLessThan);
690 }
691 return false;
692 }
693
694 // phase two: delete files until cache is under maximum allowed size
695
696 // TODO: delete files larger than allowed for a single file
697 while (t.elapsed() < 100) {
698 if (m_totalSizeOnDisk <= g_maxCacheSize || m_fiList.isEmpty()) {
699 kDebug(7113) << "total size of cache files after cleaning is" << m_totalSizeOnDisk;
700 if (scoreboard) {
701 scoreboard->maybeRemoveStaleEntries(m_fiList);
702 scoreboard->writeOut();
703 }
704 qDeleteAll(m_fiList);
705 m_fiList.clear();
706 return true;
707 }
708 CacheFileInfo *fi = m_fiList.takeFirst();
709 QString filename = filePath(fi->baseName);
710 if (QFile::remove(filename)) {
711 m_totalSizeOnDisk -= fi->sizeOnDisk;
712 if (scoreboard) {
713 scoreboard->remove(fi->baseName);
714 }
715 }
716 delete fi;
717 }
718 return false;
719 }
720
721private:
722 QStringList m_fileNameList;
723 QList<CacheFileInfo *> m_fiList;
724 qint64 m_totalSizeOnDisk;
725};
726
727
728extern "C" KDE_EXPORT int kdemain(int argc, char **argv)
729{
730 KCmdLineArgs::init(argc, argv, appName, "kdelibs4",
731 ki18n("KDE HTTP cache maintenance tool"), version,
732 ki18n("KDE HTTP cache maintenance tool"), KCmdLineArgs::CmdLineArgNone);
733
734 KCmdLineOptions options;
735 options.add("clear-all", ki18n("Empty the cache"));
736 options.add("file-info <filename>", ki18n("Display information about cache file"));
737
738 KCmdLineArgs::addCmdLineOptions( options );
739 KCmdLineArgs *args = KCmdLineArgs::parsedArgs();
740 KComponentData componentData(appName);
741
742 // we need a QCoreApplication so QCoreApplication::processEvents() works as intended
743 QCoreApplication app(argc, argv);
744
745 OperationMode mode = CleanCache;
746 if (args->isSet("clear-all")) {
747 mode = DeleteCache;
748 } else if (args->isSet("file-info")) {
749 mode = FileInfo;
750 }
751
752 // file info mode: no scanning of directories, just output info and exit.
753 if (mode == FileInfo) {
754 CacheFileInfo fi;
755 if (!readCacheFile(args->getOption("file-info"), &fi, mode)) {
756 return 1;
757 }
758 fi.prettyPrint();
759 return 0;
760 }
761
762 // make sure we're the only running instance of the cleaner service
763 if (mode == CleanCache) {
764 if (!QDBusConnection::sessionBus().isConnected()) {
765 QDBusError error(QDBusConnection::sessionBus().lastError());
766 fprintf(stderr, "%s: Could not connect to D-Bus! (%s: %s)\n", appName,
767 qPrintable(error.name()), qPrintable(error.message()));
768 return 1;
769 }
770
771 if (!QDBusConnection::sessionBus().registerService(appFullName)) {
772 fprintf(stderr, "%s: Already running!\n", appName);
773 return 0;
774 }
775 }
776
777
778 g_currentDate = time(0);
779 g_maxCacheAge = KProtocolManager::maxCacheAge();
780 g_maxCacheSize = mode == DeleteCache ? -1 : KProtocolManager::maxCacheSize() * 1024;
781
782 QString cacheDirName = KGlobal::dirs()->saveLocation("cache", "http");
783 QDir cacheDir(cacheDirName);
784 if (!cacheDir.exists()) {
785 fprintf(stderr, "%s: '%s' does not exist.\n", appName, qPrintable(cacheDirName));
786 return 0;
787 }
788
789 removeOldFiles();
790
791 if (mode == DeleteCache) {
792 QTime t;
793 t.start();
794 cacheDir.refresh();
795 //qDebug() << "time to refresh the cacheDir QDir:" << t.elapsed();
796 CacheCleaner cleaner(cacheDir);
797 while (!cleaner.processSlice()) { }
798 return 0;
799 }
800
801 QLocalServer lServer;
802 QString socketFileName = KStandardDirs::locateLocal("socket", "kio_http_cache_cleaner");
803 // we need to create the file by opening the socket, otherwise it won't work
804 QFile::remove(socketFileName);
805 lServer.listen(socketFileName);
806 QList<QLocalSocket *> sockets;
807 qint64 newBytesCounter = LLONG_MAX; // force cleaner run on startup
808
809 Scoreboard scoreboard;
810 CacheCleaner *cleaner = 0;
811 while (true) {
812 g_currentDate = time(0);
813 if (cleaner) {
814 QCoreApplication::processEvents(QEventLoop::AllEvents, 100);
815 } else {
816 // We will not immediately know when a socket was disconnected. Causes:
817 // - WaitForMoreEvents does not make processEvents() return when a socket disconnects
818 // - WaitForMoreEvents *and* a timeout is not possible.
819 QCoreApplication::processEvents(QEventLoop::WaitForMoreEvents);
820 }
821 if (!lServer.isListening()) {
822 return 1;
823 }
824 lServer.waitForNewConnection(1);
825
826 while (QLocalSocket *sock = lServer.nextPendingConnection()) {
827 sock->waitForConnected();
828 sockets.append(sock);
829 }
830
831 for (int i = 0; i < sockets.size(); i++) {
832 QLocalSocket *sock = sockets[i];
833 if (sock->state() != QLocalSocket::ConnectedState) {
834 if (sock->state() != QLocalSocket::UnconnectedState) {
835 sock->waitForDisconnected();
836 }
837 delete sock;
838 sockets.removeAll(sock);
839 i--;
840 continue;
841 }
842 sock->waitForReadyRead(0);
843 while (true) {
844 QByteArray recv = sock->read(80);
845 if (recv.isEmpty()) {
846 break;
847 }
848 Q_ASSERT(recv.size() == 80);
849 newBytesCounter += scoreboard.runCommand(recv);
850 }
851 }
852
853 // interleave cleaning with serving ioslaves to reduce "garbage collection pauses"
854 if (cleaner) {
855 if (cleaner->processSlice(&scoreboard)) {
856 // that was the last slice, done
857 delete cleaner;
858 cleaner = 0;
859 }
860 } else if (newBytesCounter > (g_maxCacheSize / 8)) {
861 cacheDir.refresh();
862 cleaner = new CacheCleaner(cacheDir);
863 newBytesCounter = 0;
864 }
865 }
866 return 0;
867}
qHash
uint qHash(const KConfigIniBackend::BufferFragment &fragment)
KCmdLineArgs
KCmdLineArgs::isSet
bool isSet(const QByteArray &option) const
KCmdLineArgs::CmdLineArgNone
CmdLineArgNone
KCmdLineArgs::init
static void init(const KAboutData *about)
KCmdLineArgs::parsedArgs
static KCmdLineArgs * parsedArgs(const QByteArray &id=QByteArray())
KCmdLineArgs::addCmdLineOptions
static void addCmdLineOptions(const KCmdLineOptions &options, const KLocalizedString &name=KLocalizedString(), const QByteArray &id=QByteArray(), const QByteArray &afterId=QByteArray())
KCmdLineArgs::getOption
QString getOption(const QByteArray &option) const
KCmdLineOptions
KCmdLineOptions::add
KCmdLineOptions & add(const KCmdLineOptions &options)
KComponentData
KDateTime
KDateTime::toString
QString toString(const QString &format) const
KDateTime::ISODate
ISODate
KDateTime::setTime_t
void setTime_t(qint64 seconds)
KProtocolManager::maxCacheSize
static int maxCacheSize()
KProtocolManager::maxCacheAge
static int maxCacheAge()
KStandardDirs::saveLocation
QString saveLocation(const char *type, const QString &suffix=QString(), bool create=true) const
KStandardDirs::locateLocal
static QString locateLocal(const char *type, const QString &filename, bool createDir, const KComponentData &cData=KGlobal::mainComponent())
QHash
QIODevice
QList
QSet
header
const char header[]
kDebug
#define kDebug
readCommand
static CacheCleanerCommand readCommand(const QByteArray &cmd, CacheFileInfo *fi)
Definition: http_cache_cleaner.cpp:378
OperationMode
OperationMode
Definition: http_cache_cleaner.cpp:158
CleanCache
@ CleanCache
Definition: http_cache_cleaner.cpp:159
DeleteCache
@ DeleteCache
Definition: http_cache_cleaner.cpp:160
FileInfo
@ FileInfo
Definition: http_cache_cleaner.cpp:161
s_hashedUrlBits
static const int s_hashedUrlBits
Definition: http_cache_cleaner.cpp:61
qHash
uint qHash(const CacheIndex &ci)
Definition: http_cache_cleaner.cpp:372
appName
static const char appName[]
Definition: http_cache_cleaner.cpp:56
g_currentDate
time_t g_currentDate
Definition: http_cache_cleaner.cpp:51
kdemain
int kdemain(int argc, char **argv)
Definition: http_cache_cleaner.cpp:728
filenameFromUrl
static QString filenameFromUrl(const QByteArray &url)
Definition: http_cache_cleaner.cpp:201
CacheCleanerCommand
CacheCleanerCommand
Definition: http_cache_cleaner.cpp:265
UpdateFileCommand
@ UpdateFileCommand
Definition: http_cache_cleaner.cpp:268
InvalidCommand
@ InvalidCommand
Definition: http_cache_cleaner.cpp:266
CreateFileNotificationCommand
@ CreateFileNotificationCommand
Definition: http_cache_cleaner.cpp:267
CacheFileInfoPtrLessThan
bool CacheFileInfoPtrLessThan(const CacheFileInfo *cf1, const CacheFileInfo *cf2)
Definition: http_cache_cleaner.cpp:153
g_maxCacheAge
int g_maxCacheAge
Definition: http_cache_cleaner.cpp:52
appFullName
static const char appFullName[]
Definition: http_cache_cleaner.cpp:55
readLineChecked
static bool readLineChecked(QIODevice *dev, QByteArray *line)
Definition: http_cache_cleaner.cpp:217
readCacheFile
static bool readCacheFile(const QString &baseName, CacheFileInfo *fi, OperationMode mode)
Definition: http_cache_cleaner.cpp:271
s_hashedUrlBytes
static const int s_hashedUrlBytes
Definition: http_cache_cleaner.cpp:63
readTextHeader
static bool readTextHeader(QFile *file, CacheFileInfo *fi, OperationMode mode)
Definition: http_cache_cleaner.cpp:229
version
static const char version[]
Definition: http_cache_cleaner.cpp:65
removeOldFiles
static void removeOldFiles()
Definition: http_cache_cleaner.cpp:607
filePath
static QString filePath(const QString &baseName)
Definition: http_cache_cleaner.cpp:208
dateString
static QString dateString(qint64 date)
Definition: http_cache_cleaner.cpp:87
g_maxCacheSize
qint64 g_maxCacheSize
Definition: http_cache_cleaner.cpp:53
readBinaryHeader
static bool readBinaryHeader(const QByteArray &d, CacheFileInfo *fi)
Definition: http_cache_cleaner.cpp:171
timeSizeFits
static bool timeSizeFits(qint64 intTime)
Definition: http_cache_cleaner.cpp:164
s_hashedUrlNibbles
static const int s_hashedUrlNibbles
Definition: http_cache_cleaner.cpp:62
kcmdlineargs.h
kcomponentdata.h
operator==
bool operator==(const KEntry &k1, const KEntry &k2)
operator<
bool operator<(const KEntryKey &k1, const KEntryKey &k2)
kdatetime.h
kdebug.h
kglobal.h
klocale.h
ki18n
KLocalizedString ki18n(const char *msg)
kprotocolmanager.h
kstandarddirs.h
KGlobal::dirs
KStandardDirs * dirs()
add
void add(const QString &fileClass, const QString &directory)
remove
KGuiItem remove()
ok
KGuiItem ok()
This file is part of the KDE documentation.
Documentation copyright © 1996-2023 The KDE developers.
Generated on Mon Feb 20 2023 00:00:00 by doxygen 1.9.6 written by Dimitri van Heesch, © 1997-2006

KDE's Doxygen guidelines are available online.

KIOSlave

Skip menu "KIOSlave"
  • Main Page
  • Alphabetical List
  • Class List
  • Class Hierarchy
  • Class Members
  • File List
  • File Members
  • Related Pages

kdelibs-4.14.38 API Reference

Skip menu "kdelibs-4.14.38 API Reference"
  • DNSSD
  • Interfaces
  •   KHexEdit
  •   KMediaPlayer
  •   KSpeech
  •   KTextEditor
  • kconf_update
  • KDE3Support
  •   KUnitTest
  • KDECore
  • KDED
  • KDEsu
  • KDEUI
  • KDEWebKit
  • KDocTools
  • KFile
  • KHTML
  • KImgIO
  • KInit
  • kio
  • KIOSlave
  • KJS
  •   KJS-API
  •   WTF
  • kjsembed
  • KNewStuff
  • KParts
  • KPty
  • Kross
  • KUnitConversion
  • KUtils
  • Nepomuk
  • Plasma
  • Solid
  • Sonnet
  • ThreadWeaver
Report problems with this website to our bug tracking system.
Contact the specific authors with questions and comments about the page contents.

KDE® and the K Desktop Environment® logo are registered trademarks of KDE e.V. | Legal