Add a couple of callgrind tests.

Turn off tests >2000 messages.
author: Rohan McGovern <rohan.mcgovern@nokia.com> 2009-03-30 13:22:56 +1000
committer: Rohan McGovern <rohan.mcgovern@nokia.com> 2009-03-30 13:22:56 +1000
commit: 92ada7b74ccf11e0e66ea2b161be3cc33f52643a (patch)
tree: 5ca4e49822d6de5744cf426cd8e89b9cbda44acc /benchmarks
parent: 39caf73be2e22d2c1f11d827dee019964ef69d48 (diff)
2 files changed, 167 insertions, 14 deletions
diff --git a/benchmarks/tst_messageserver/benchmarkcontext.cpp b/benchmarks/tst_messageserver/benchmarkcontext.cpp
index dba01d0e..4c57ecd8 100644
--- a/benchmarks/tst_messageserver/benchmarkcontext.cpp
+++ b/benchmarks/tst_messageserver/benchmarkcontext.cpp
@@ -12,6 +12,7 @@
 #include "testfsusage.h"
 #include "testmalloc.h"
 #include "3rdparty/cycle_p.h"
+#include "3rdparty/callgrind_p.h"
 #include "3rdparty/valgrind_p.h"
 
 #include <qmailnamespace.h>
@@ -20,6 +21,8 @@
 #include <QDir>
 #include <QTest>
 
+#undef HAVE_TICK_COUNTER // not useful for this test
+
 class BenchmarkContextPrivate
 {
 public:
@@ -45,12 +48,69 @@ BenchmarkContext::BenchmarkContext(bool xml)
 
     TestMalloc::resetNow();
     TestMalloc::resetPeak();
+    CALLGRIND_ZERO_STATS;
 }
 
 BenchmarkContext::~BenchmarkContext()
 {
     if (!QTest::currentTestFailed()) {
-        qint64 newQmfUsage = TestFsUsage::usage(QMail::dataPath());
+        CALLGRIND_DUMP_STATS;
+
+        if (RUNNING_ON_VALGRIND) {
+            // look for callgrind.out.<PID>.<NUM> in CWD
+            QDir dir;
+            QFile file;
+            QString match = QString("callgrind.out.%1.").arg(::getpid());
+            foreach (QString const& entry, dir.entryList(QDir::Files,QDir::Name)) {
+                if (entry.startsWith(match)) {
+                    file.setFileName(entry);
+                }
+            }
+
+            if (file.fileName().isEmpty()) {
+                qWarning("I'm running on valgrind, but couldn't find callgrind.out file...");
+                return;
+            }
+
+            if (!file.open(QIODevice::ReadOnly)) {
+                qWarning("Failed to open %s: %s", qPrintable(file.fileName()),
+                    qPrintable(file.errorString()));
+            }
+
+            QList<QByteArray> lines = file.readAll().split('\n');
+            qint64 ir = -1;
+            bool is_ir = false;
+            foreach (QByteArray const& line, lines) {
+                if (line == "events: Ir") {
+                    is_ir = true;
+                    continue;
+                }
+                if (is_ir) {
+                    if (line.startsWith("summary: ")) {
+                        bool ok;
+                        ir = line.mid(sizeof("summary: ")-1).toLongLong(&ok);
+                        if (!ok) ir = -1;
+                    }
+                    is_ir = false;
+                    break;
+                }
+            }
+            if (ir == -1) {
+                qWarning("Found callgrind.out file, but it doesn't seem to contain a valid "
+                    "Ir count...");
+                return;
+            }
+
+            if (d->xml) {
+                fprintf(stdout, "<BenchmarkResult metric=\"callgrind\" tag=\"%s\" value=\"%lld\" iterations=\"1\"/>\n", QTest::currentDataTag(), ir);
+            }
+            else {
+                qWarning("callgrind instruction count: %lld", ir);
+            }
+            // callgrind invalidates rest of results
+            return;
+        }
+
 #ifdef HAVE_TICK_COUNTER
         CycleCounterTicks newTicks = getticks();
 #endif
@@ -59,6 +119,7 @@ BenchmarkContext::~BenchmarkContext()
         int heapUsageTotal  = TestMalloc::peakTotal()/1000;
         int heapUsageUsable = TestMalloc::peakUsable()/1000;
         int ms = d->time.elapsed();
+        qint64 newQmfUsage = TestFsUsage::usage(QMail::dataPath());
 #ifdef HAVE_TICK_COUNTER
         quint64 cycles = quint64(elapsed(newTicks,d->ticks));
 #endif
diff --git a/benchmarks/tst_messageserver/tst_messageserver.cpp b/benchmarks/tst_messageserver/tst_messageserver.cpp
index e1ab74da..96a28055 100644
--- a/benchmarks/tst_messageserver/tst_messageserver.cpp
+++ b/benchmarks/tst_messageserver/tst_messageserver.cpp
@@ -8,11 +8,8 @@
 **
 ****************************************************************************/
 
-#include <QProcess>
-#include <QTemporaryFile>
 #include <QTest>
-#include <QDir>
-#include <QTimer>
+#include <QtCore>
 #include <errno.h>
 #include <qmailnamespace.h>
 #include <stdio.h>
@@ -76,6 +73,7 @@ private:
     void addAccount(QMailAccount*, QString const&, QString const&, QString const&, QString const&, int);
     void removePath(QString const&);
     void runInChildProcess(TestFunction);
+    void runInCallgrind(QString const&);
 
     QEventLoop*                  m_loop;
     QTimer*                      m_timer;
@@ -148,6 +146,100 @@ void tst_MessageServer::removePath(QString const& path)
 void tst_MessageServer::completeRetrievalImap()
 {
     runInChildProcess(&tst_MessageServer::completeRetrievalImap_impl);
+    if (QTest::currentTestFailed()) return;
+
+    QByteArray tag = QTest::currentDataTag();
+    if (tag == "small_messages--100" || tag == "small_messages--500") {
+        runInCallgrind("completeRetrievalImap:" + tag);
+    }
+}
+
+void tst_MessageServer::runInCallgrind(QString const& testfunc)
+{
+    if (RUNNING_ON_VALGRIND) return;
+
+    /*
+        Run a particular testfunc in a separate process under callgrind.
+    */
+    QString thisapp = QCoreApplication::applicationFilePath();
+
+    // Strip any testfunction args
+    QMetaObject const* mo = metaObject();
+    QStringList testfunctions;
+    for (int i = 0; i < mo->methodCount(); ++i) {
+        QMetaMethod mm = mo->method(i);
+        if (mm.methodType() == QMetaMethod::Slot && mm.access() == QMetaMethod::Private) {
+            QByteArray sig = mm.signature();
+            testfunctions << QString::fromLatin1(sig.left(sig.indexOf('(')));
+        }
+    }
+    QStringList args;
+    foreach (QString const& arg, QCoreApplication::arguments()) {
+        bool bad = false;
+        foreach (QString const& tf, testfunctions) {
+            if (arg.startsWith(tf)) {
+                bad = true;
+                break;
+            }
+        }
+        if (!bad) args << arg;
+    }
+    args.removeAt(0); // application name
+
+    QProcess proc;
+    proc.setProcessChannelMode(QProcess::MergedChannels);
+
+    proc.start("valgrind", QStringList()
+        << "--tool=callgrind"
+        << "--quiet"
+        << "--"
+        << thisapp
+        << args
+        << testfunc
+    );
+
+    if (!proc.waitForStarted(30000)) {
+        QFAIL(qPrintable(QString("Failed to start %1 under callgrind: %2").arg(testfunc)
+            .arg(proc.errorString())));
+    }
+
+    static const int timeoutSeconds = 60*60;
+    if (!proc.waitForFinished(timeoutSeconds*1000)) {
+        QFAIL(qPrintable(QString("%1 in callgrind didn't finish within %2 seconds\n"
+            "Output:\n%3")
+            .arg(testfunc)
+            .arg(timeoutSeconds)
+            .arg(QString::fromLocal8Bit(proc.readAll()))
+        ));
+    }
+
+    if (proc.exitStatus() != QProcess::NormalExit) {
+        QFAIL(qPrintable(QString("%1 in callgrind crashed: %2\n"
+            "Output:\n%3")
+            .arg(testfunc)
+            .arg(proc.errorString())
+            .arg(QString::fromLocal8Bit(proc.readAll()))
+        ));
+    }
+
+    if (proc.exitCode() != 0) {
+        QFAIL(qPrintable(QString("%1 in callgrind exited with code %2\n"
+            "Output:\n%3")
+            .arg(testfunc)
+            .arg(proc.exitCode())
+            .arg(QString::fromLocal8Bit(proc.readAll()))
+        ));
+    }
+
+    /*
+        OK, it ran fine.  Now reproduce the benchmark lines exactly as they appeared in
+        the child process.
+    */
+    foreach (QByteArray const& line, proc.readAll().split('\n')) {
+        if (line.contains("callgrind")) {
+            fprintf(stdout, "%s\n", line.constData());
+        }
+    }
 }
 
 void tst_MessageServer::runInChildProcess(TestFunction fn)
@@ -420,28 +512,28 @@ void tst_MessageServer::completeRetrievalImap_data()
     ;
 
     list.clear();
-    for (int i = 0; i < 5000; ++i)
+    for (int i = 0; i < 2000; ++i)
         list << TestMail();
-    QTest::newRow("small_messages--5000")
-        << QString::fromLatin1("mailtst35")
-        << QString::fromLatin1("testme35")
+    QTest::newRow("small_messages--2000")
+        << QString::fromLatin1("mailtst34")
+        << QString::fromLatin1("testme34")
         << m_imapServer
         << 143
         << list
     ;
 
+#if VERY_PATIENT_TESTER
     list.clear();
-    for (int i = 0; i < 2000; ++i)
+    for (int i = 0; i < 5000; ++i)
         list << TestMail();
-    QTest::newRow("small_messages--2000")
-        << QString::fromLatin1("mailtst34")
-        << QString::fromLatin1("testme34")
+    QTest::newRow("small_messages--5000")
+        << QString::fromLatin1("mailtst35")
+        << QString::fromLatin1("testme35")
         << m_imapServer
         << 143
         << list
     ;
 
-#if VERY_PATIENT_TESTER
     list.clear();
     for (int i = 0; i < 10000; ++i)
         list << TestMail();
author	Rohan McGovern <rohan.mcgovern@nokia.com>	2009-03-30 13:22:56 +1000
committer	Rohan McGovern <rohan.mcgovern@nokia.com>	2009-03-30 13:22:56 +1000
commit	92ada7b74ccf11e0e66ea2b161be3cc33f52643a (patch)
tree	5ca4e49822d6de5744cf426cd8e89b9cbda44acc /benchmarks
parent	39caf73be2e22d2c1f11d827dee019964ef69d48 (diff)