Implement most missing functions

src/libdanbooru/danbooruservice.cpp

@@ -20,30 +20,34 @@
  *
  */
 
+// Qt
+
 #include <QtCore/QDebug>
 #include <qjson/parser.h>
 
+// KDE
+
 #include <KIO/Job>
 #include <KIO/Scheduler>
 
-#include "utils.h"
+// Own
 
 #include "danbooruservice.h"
 #include "danboorupost.h"
 #include "danboorupool.h"
-
+#include "danboorutag.h"
+#include "utils.h"
 
 namespace Danbooru {
 
     using KIO::StoredTransferJob;
 
     const QString DanbooruService::POST_URL ="post/index.json" ;
-    const QString DanbooruService::TAG_URL = "tag/index.json";
+    const QString DanbooruService::TAG_URL = "tag/index.xml";
     const QString DanbooruService::POOL_URL = "pool/index.json";
     const QString DanbooruService::ARTIST_URL = "artist/index.json";
     const QString DanbooruService::POOL_DATA_URL = "pool/show.xml";
     const QString DanbooruService::RELATED_TAG_URL = "tag/related.json";
-    const QMap<QString, QStringList> DanbooruService::RATINGS = initRatings();
 
     DanbooruService::DanbooruService(KUrl& boardUrl, QString username,
                                      QString password,
@@ -52,7 +56,7 @@ namespace Danbooru {
                                      m_url(boardUrl),
                                      m_username(username),
                                      m_password(password),
-                                     m_maxRating("Safe"),
+                                     m_maxRating(Safe),
                                      m_currentPosts(0)
     {
     }
@@ -62,26 +66,6 @@ namespace Danbooru {
 
     }
 
-    const QMap< QString, QStringList > DanbooruService::initRatings()
-    {
-        QMap<QString, QStringList> map;
-
-        QStringList safeRatings;
-        safeRatings.append(QString("Safe"));
-
-        QStringList questionableRatings = QStringList(safeRatings);
-        questionableRatings.append(QString("Questionable"));
-
-        QStringList explicitRatings = QStringList(questionableRatings);
-        explicitRatings.append(QString("Explicit"));
-
-        map.insert("Safe", safeRatings);
-        map.insert("Questionable", questionableRatings);
-        map.insert("Explicit", explicitRatings);
-
-        return map;
-
-    }
 
     void DanbooruService::getPostList(int page, QStringList tags, int limit)
     {
@@ -90,19 +74,22 @@ namespace Danbooru {
 
         limit = limit > 100 ? 100: limit;
 
-        QMap<QString, QString> map;
+        QMap<QString, QString> parameters;
 
-        map.insert("limit", QString::number(limit));
+        parameters.insert("limit", QString::number(limit));
-        map.insert("page", QString::number(page));
+        parameters.insert("page", QString::number(page));
 
         KUrl danbooruUrl = requestUrl(m_url, POST_URL, m_username,
-                                     m_password, map, tags);
+                                     m_password, parameters, tags);
 
         qDebug() << "Final constructed URL" << danbooruUrl.url();
 
         KIO::StoredTransferJob* job = KIO::storedGet(danbooruUrl, KIO::NoReload,
                                                      KIO::HideProgressInfo);
 
+        // This job can use JSON data
+        job->setProperty("needsXML", false);
+
         connect(job, SIGNAL(result(KJob*)), this,
                 SLOT(processPostList(KJob*)));
 
@@ -110,12 +97,51 @@ namespace Danbooru {
 
     void DanbooruService::getTagList(int limit, QString name)
     {
+        QMap<QString, QString> parameters;
+        parameters.insert("limit", QString::number(limit));
 
+        if (!name.isEmpty()) {
+            parameters.insert("name", name);
+        }
+
+        KUrl danbooruUrl = requestUrl(m_url, TAG_URL, m_username, m_password,
+                                      parameters);
+        qDebug() << "Final constructed URL" << danbooruUrl.url();
+
+        KIO::StoredTransferJob* job = KIO::storedGet(danbooruUrl, KIO::NoReload,
+                                                     KIO::HideProgressInfo);
+
+        connect(job, SIGNAL(result(KJob*)), this, SLOT(processTagList(KJob*)));
     }
 
     void DanbooruService::getPool(int poolId, int page)
     {
 
+        QMap<QString, QString> parameters;
+
+        parameters.insert("id", QString::number(poolId));
+
+        if (page > 1) {
+            parameters.insert("page", QString::number(page));
+        }
+
+        KUrl danbooruUrl = requestUrl(m_url, POOL_DATA_URL, m_username,
+                                      m_password, parameters);
+
+        qDebug() << "Final constructed URL" << danbooruUrl.url();
+
+
+        KIO::StoredTransferJob* job = KIO::storedGet(danbooruUrl, KIO::NoReload,
+                                                     KIO::HideProgressInfo);
+
+        //HACK: Most Danbooru implementations don't provide valid data on
+        // pools via JSON, hence we set XML and parse the XML data.
+
+        job->setProperty("needsXML", true);
+
+        connect(job, SIGNAL(result(KJob*)), this,
+                SLOT(processPostList(KJob*)));
+
     }
 
     void DanbooruService::getPoolList(int page)
@@ -137,40 +163,81 @@ namespace Danbooru {
 
         KIO::StoredTransferJob* job = KIO::storedGet(danbooruUrl, KIO::NoReload,
                                                      KIO::HideProgressInfo);
+        // This job can use JSON data
+        job->setProperty("needsXML", false);
 
         connect(job, SIGNAL(result(KJob*)), this,
                 SLOT(processPoolList(KJob*)));
 
     }
 
-    void DanbooruService::getRelatedTags(QStringList tags,
+    void DanbooruService::getRelatedTags(QStringList tags, Danbooru::TagType tagType)
-                                         Danbooru::TagType tagType)
     {
 
     }
 
     // Getters / setters
 
-    QStringList DanbooruService::blacklist() const
+    void DanbooruService::setBlacklist(const QSet< QString >& blacklist)
+    {
+
+        if (!blacklist.isEmpty()) {
+            m_blacklist = blacklist;
+        }
+
+    }
+
+
+    const QSet< QString > DanbooruService::blacklist() const
     {
         return m_blacklist;
     }
 
     const QStringList DanbooruService::allowedRatings() const
     {
-        return RATINGS.value(m_maxRating);
+        QStringList ratings;
-    }
 
-    void DanbooruService::setMaximumAllowedRating(QString rating)
+        if (m_maxRating.testFlag(Safe)) {
-    {
+            ratings.append("Safe");
-        if (RATINGS.contains(rating)) {
-            m_maxRating = rating;
         }
+
+        if (m_maxRating.testFlag(Questionable)) {
+            ratings.append("Questionable");
+        }
+
+        if (m_maxRating.testFlag(Explicit)) {
+            ratings.append("Explicit");
+        }
+
+        return ratings;
+
     }
 
-    const QString DanbooruService::maximumAllowedRating() const
+    void DanbooruService::setMaximumAllowedRating(const Danbooru::Ratings& rating)
     {
+        Ratings flags;
+
+        switch (rating) {
+            case Safe:
+                flags = Safe;
+                break;
+            case Questionable:
+                flags = Safe | Questionable;
+                break;
+            case Explicit:
+                flags = Safe | Questionable | Explicit;
+                break;
+        }
+
+        m_maxRating = flags;
+
+    }
+
+    const Ratings DanbooruService::maximumAllowedRating() const
+    {
+
         return m_maxRating;
+
     }
 
     // Slots
@@ -194,29 +261,66 @@ namespace Danbooru {
 
         QByteArray data = jobResult->data();
 
-        QJson::Parser parser;
+        qDebug() << jobResult->mimetype();
 
         bool ok;
 
-        QVariant result = parser.parse(data, &ok);
+        bool needsXML = job->property("needsXML").toBool();
+
+        QList<QVariant> postList;
+
+        if (needsXML) {
+            // Special cases for pools
+            postList = parseDanbooruResult(data, QString("pool"), &ok);
+        } else {
+            postList = parseDanbooruResult(data, &ok);
+        }
 
         if (!ok) {
             Q_EMIT(downloadError(QString("Unable to decode data")));
             return;
         }
 
-        QList<QVariant> postList = result.toList();
-
         // How many posts do we have to fetch?
 
         m_currentPosts = postList.length();
 
-        Q_FOREACH(QVariant element, postList) {
+        Q_FOREACH(const QVariant& element, postList) {
 
             QVariantMap map = element.toMap();
 
             DanbooruPost* post = new DanbooruPost(map);
 
+//             qDebug() << "Got rating" << post->rating();
+
+            // First check, for rating
+
+            if (post->rating() > m_maxRating) {
+                qDebug() << "Skipping " << post->fileUrl();
+                qDebug() << "Rating was " << post->rating();
+                m_currentPosts--;
+                delete post;
+                continue;
+            }
+
+            // second check, blacklist
+            // We make a copy due to the fact that otherwise intersect()
+            // will change the set in place
+
+            QSet<QString> temp = m_blacklist;
+
+            QSet<QString> overlap = temp.intersect(post->tags());
+
+            if (!overlap.isEmpty()) {
+
+                // Blacklisted tags are present, do not use this post
+                qDebug() << "Skipping " << post->fileUrl();
+                qDebug() << "Blacklisted tags " << overlap;
+                m_currentPosts--;
+                delete post;
+                continue;
+            }
+
             StoredTransferJob* pixmapJob = KIO::storedGet(post->thumbnailUrl(),
                 KIO::NoReload, KIO::HideProgressInfo);
 
@@ -238,6 +342,46 @@ namespace Danbooru {
 
     }
 
+    void DanbooruService::processTagList(KJob* job)
+    {
+        qDebug() << "Got tag data OK";
+
+        if (job->error()) {
+            qDebug() << "Something went wrong";
+            Q_EMIT(downloadError(job->errorString()));
+        }
+
+        StoredTransferJob* jobResult = qobject_cast<StoredTransferJob*>(job);
+
+        if (jobResult == 0) {
+            Q_EMIT(downloadError(QString("Internal error")));
+            return;
+        }
+
+        QByteArray data = jobResult->data();
+
+        bool ok;
+
+        // Most Danbooru implementations return tags in wrong order when
+        // using JSON, so we have to fall back to XML
+        QList<QVariant> tagList = parseDanbooruResult(data, "tag", &ok);
+
+        if (!ok) {
+            qDebug() << "Something went wrong";
+            Q_EMIT(downloadError(QString("Unable to decode data")));
+            return;
+        }
+
+        qDebug() << "Populating tag list, length" << tagList.length();
+
+        Q_FOREACH(const QVariant& element, tagList) {
+            QVariantMap map = element.toMap();
+            DanbooruTag* tag = new DanbooruTag(map);
+            Q_EMIT(tagDownloaded(tag));
+        }
+    }
+
+
     void DanbooruService::processRelatedTagList(KJob* job)
     {
         Q_UNUSED(job)
@@ -262,22 +406,16 @@ namespace Danbooru {
 
         QByteArray data = jobResult->data();
 
-        QJson::Parser parser;
-
         bool ok;
 
-        QVariant result = parser.parse(data, &ok);
+        QList<QVariant> poolList = parseDanbooruResult(data,  &ok);
 
         if (!ok) {
             Q_EMIT(downloadError(QString("Unable to decode data")));
             return;
         }
 
-        QList<QVariant> poolList = result.toList();
+        Q_FOREACH(const QVariant& element, poolList) {
-
-        int currentPools = poolList.length();
-
-        Q_FOREACH(QVariant element, poolList) {
             QVariantMap map = element.toMap();
 
             DanbooruPool* pool = new DanbooruPool(map);
@@ -335,6 +473,5 @@ namespace Danbooru {
 
     }
 
-
 } // namespace Danbooru
 

src/libdanbooru/danbooruservice.h

@@ -31,42 +31,32 @@
  *
 **/
 
+// Qt
+
 #include <QtCore/QObject>
 #include <QtCore/QString>
 #include <QtCore/QStringList>
+#include <QtCore/QSet>
+
+// KDE
 
 #include <KUrl>
 #include <KIO/JobClasses>
 
+// Own
+
+#include "danbooru.h"
+
 class QPixmap;
 class KUrl;
 class KJob;
 
-/**
- * @brief The Danbooru namespace.
- *
- **/
-namespace Danbooru {
 
-    /**
+namespace Danbooru {
-     * @brief Types of tags
-     *
-     * A Danbooru tag is not simply a string, but carries some (limited)
-     * semantic information. In particular, tags are organized in what they
-     * refer to, either something related to the image itself, or to the
-     * artist that drew it, or the copyrights associated to the image, or even
-     * the characters that are represented in it.
-     *
-     **/
-    enum TagType {
-        General, /**< Generic tags **/
-        Artist, /**< Tags related to artists **/
-        Copyright, /**<Tags related to copyrights **/
-        Character /**<Tags related to characters **/
-    };
 
     class DanbooruPost;
     class DanbooruPool;
+    class DanbooruTag;
 
     using KIO::StoredTransferJob;
 
@@ -96,29 +86,16 @@ namespace Danbooru {
         static const QString POOL_DATA_URL;
         static const QString RELATED_TAG_URL;
 
-        // Ratings
-
-        static const QMap<QString, QStringList> RATINGS;
-
         // member variables
 
         KUrl m_url;
         QString m_username;
         QString m_password;
-        QStringList m_blacklist;
+        QSet<QString> m_blacklist;
-        QString m_maxRating;
+        Ratings m_maxRating;
 
         unsigned int m_currentPosts; // To tell when to quit
 
-        // private functions
-
-        /**
-         * @internal
-         *
-         **/
-        static const QMap< QString, QStringList > initRatings();
-
-
     public:
 
         /**
@@ -169,10 +146,15 @@ namespace Danbooru {
         /**
          * @brief Get a list of tags from the board
          *
+         * If name is supplied, a list of tags including the exact name of the
+         * tag is fetched from Danbooru, otherwise the most recent tags are
+         * retrieved.
+         *
          * The tagDownloaded signal is emitted every time a tag has been
          * retrieved.
          *
          * @param limit The number of tags to get.
+         * @param name The name of the tag to retrieve, or an empty string
          *
          **/
         void getTagList(int limit=10, QString name="");
@@ -184,7 +166,7 @@ namespace Danbooru {
          * @param tagType The type of tag to query for
          **/
         void getRelatedTags(QStringList tags=QStringList(),
-                            TagType tagType=General);
+                            Danbooru::TagType tagType=General);
 
         /**
          * @return The currently allowed ratings when downloading posts.
@@ -194,12 +176,14 @@ namespace Danbooru {
         /**
          * @return The maximum allowed rating for a post.
          **/
-        const QString maximumAllowedRating() const;
+        const Ratings maximumAllowedRating() const;
 
         /**
          * @return The currently blacklisted tags.
          **/
-        QStringList blacklist() const;
+        const QSet<QString> blacklist() const;
+
+        void setBlacklist(const QSet<QString>& blacklist);
 
         /**
          * @brief Set the maximum allowed rating for the board.
@@ -208,7 +192,7 @@ namespace Danbooru {
          * downloaded.
          *
          **/
-        void setMaximumAllowedRating(QString rating);
+        void setMaximumAllowedRating(const Ratings& rating);
 
         /**
          * @brief Set the tag blacklist.
@@ -221,6 +205,7 @@ namespace Danbooru {
     private Q_SLOTS:
         void processPostList(KJob* job);
         void processPoolList(KJob* job);
+        void processTagList(KJob* job);
         void processRelatedTagList(KJob* job);
         void downloadThumbnail(KJob* job);
         void downloadAllTags(KJob* job);
@@ -258,6 +243,8 @@ namespace Danbooru {
         void postDownloaded(Danbooru::DanbooruPost* post);
 
         void poolDownloaded(Danbooru::DanbooruPool* pool);
+
+        void tagDownloaded(Danbooru::DanbooruTag* tag);
         // TODO: Tags and similar
 
     };