From 869620d6f628509f3efaf29437f2712d61cef639 Mon Sep 17 00:00:00 2001 From: Luca Beltrame Date: Sun, 17 Mar 2013 12:00:30 +0100 Subject: [PATCH] Implement most missing functions --- src/libdanbooru/danbooruservice.cpp | 243 ++++++++++++++++++++++------ src/libdanbooru/danbooruservice.h | 67 ++++---- 2 files changed, 217 insertions(+), 93 deletions(-) diff --git a/src/libdanbooru/danbooruservice.cpp b/src/libdanbooru/danbooruservice.cpp index 24132a8..d5bec99 100644 --- a/src/libdanbooru/danbooruservice.cpp +++ b/src/libdanbooru/danbooruservice.cpp @@ -20,30 +20,34 @@ * */ +// Qt + #include #include +// KDE + #include #include -#include "utils.h" +// Own #include "danbooruservice.h" #include "danboorupost.h" #include "danboorupool.h" - +#include "danboorutag.h" +#include "utils.h" namespace Danbooru { using KIO::StoredTransferJob; const QString DanbooruService::POST_URL ="post/index.json" ; - const QString DanbooruService::TAG_URL = "tag/index.json"; + const QString DanbooruService::TAG_URL = "tag/index.xml"; const QString DanbooruService::POOL_URL = "pool/index.json"; const QString DanbooruService::ARTIST_URL = "artist/index.json"; const QString DanbooruService::POOL_DATA_URL = "pool/show.xml"; const QString DanbooruService::RELATED_TAG_URL = "tag/related.json"; - const QMap DanbooruService::RATINGS = initRatings(); DanbooruService::DanbooruService(KUrl& boardUrl, QString username, QString password, @@ -52,7 +56,7 @@ namespace Danbooru { m_url(boardUrl), m_username(username), m_password(password), - m_maxRating("Safe"), + m_maxRating(Safe), m_currentPosts(0) { } @@ -62,26 +66,6 @@ namespace Danbooru { } - const QMap< QString, QStringList > DanbooruService::initRatings() - { - QMap map; - - QStringList safeRatings; - safeRatings.append(QString("Safe")); - - QStringList questionableRatings = QStringList(safeRatings); - questionableRatings.append(QString("Questionable")); - - QStringList explicitRatings = QStringList(questionableRatings); - explicitRatings.append(QString("Explicit")); - - map.insert("Safe", safeRatings); - map.insert("Questionable", questionableRatings); - map.insert("Explicit", explicitRatings); - - return map; - - } void DanbooruService::getPostList(int page, QStringList tags, int limit) { @@ -90,19 +74,22 @@ namespace Danbooru { limit = limit > 100 ? 100: limit; - QMap map; + QMap parameters; - map.insert("limit", QString::number(limit)); - map.insert("page", QString::number(page)); + parameters.insert("limit", QString::number(limit)); + parameters.insert("page", QString::number(page)); KUrl danbooruUrl = requestUrl(m_url, POST_URL, m_username, - m_password, map, tags); + m_password, parameters, tags); qDebug() << "Final constructed URL" << danbooruUrl.url(); KIO::StoredTransferJob* job = KIO::storedGet(danbooruUrl, KIO::NoReload, KIO::HideProgressInfo); + // This job can use JSON data + job->setProperty("needsXML", false); + connect(job, SIGNAL(result(KJob*)), this, SLOT(processPostList(KJob*))); @@ -110,12 +97,51 @@ namespace Danbooru { void DanbooruService::getTagList(int limit, QString name) { + QMap parameters; + parameters.insert("limit", QString::number(limit)); + if (!name.isEmpty()) { + parameters.insert("name", name); + } + + KUrl danbooruUrl = requestUrl(m_url, TAG_URL, m_username, m_password, + parameters); + qDebug() << "Final constructed URL" << danbooruUrl.url(); + + KIO::StoredTransferJob* job = KIO::storedGet(danbooruUrl, KIO::NoReload, + KIO::HideProgressInfo); + + connect(job, SIGNAL(result(KJob*)), this, SLOT(processTagList(KJob*))); } void DanbooruService::getPool(int poolId, int page) { + QMap parameters; + + parameters.insert("id", QString::number(poolId)); + + if (page > 1) { + parameters.insert("page", QString::number(page)); + } + + KUrl danbooruUrl = requestUrl(m_url, POOL_DATA_URL, m_username, + m_password, parameters); + + qDebug() << "Final constructed URL" << danbooruUrl.url(); + + + KIO::StoredTransferJob* job = KIO::storedGet(danbooruUrl, KIO::NoReload, + KIO::HideProgressInfo); + + //HACK: Most Danbooru implementations don't provide valid data on + // pools via JSON, hence we set XML and parse the XML data. + + job->setProperty("needsXML", true); + + connect(job, SIGNAL(result(KJob*)), this, + SLOT(processPostList(KJob*))); + } void DanbooruService::getPoolList(int page) @@ -137,40 +163,81 @@ namespace Danbooru { KIO::StoredTransferJob* job = KIO::storedGet(danbooruUrl, KIO::NoReload, KIO::HideProgressInfo); + // This job can use JSON data + job->setProperty("needsXML", false); connect(job, SIGNAL(result(KJob*)), this, SLOT(processPoolList(KJob*))); } - void DanbooruService::getRelatedTags(QStringList tags, - Danbooru::TagType tagType) + void DanbooruService::getRelatedTags(QStringList tags, Danbooru::TagType tagType) { } // Getters / setters - QStringList DanbooruService::blacklist() const + void DanbooruService::setBlacklist(const QSet< QString >& blacklist) + { + + if (!blacklist.isEmpty()) { + m_blacklist = blacklist; + } + + } + + + const QSet< QString > DanbooruService::blacklist() const { return m_blacklist; } const QStringList DanbooruService::allowedRatings() const { - return RATINGS.value(m_maxRating); - } + QStringList ratings; - void DanbooruService::setMaximumAllowedRating(QString rating) - { - if (RATINGS.contains(rating)) { - m_maxRating = rating; + if (m_maxRating.testFlag(Safe)) { + ratings.append("Safe"); } + + if (m_maxRating.testFlag(Questionable)) { + ratings.append("Questionable"); + } + + if (m_maxRating.testFlag(Explicit)) { + ratings.append("Explicit"); + } + + return ratings; + } - const QString DanbooruService::maximumAllowedRating() const + void DanbooruService::setMaximumAllowedRating(const Danbooru::Ratings& rating) { + Ratings flags; + + switch (rating) { + case Safe: + flags = Safe; + break; + case Questionable: + flags = Safe | Questionable; + break; + case Explicit: + flags = Safe | Questionable | Explicit; + break; + } + + m_maxRating = flags; + + } + + const Ratings DanbooruService::maximumAllowedRating() const + { + return m_maxRating; + } // Slots @@ -194,29 +261,66 @@ namespace Danbooru { QByteArray data = jobResult->data(); - QJson::Parser parser; + qDebug() << jobResult->mimetype(); bool ok; - QVariant result = parser.parse(data, &ok); + bool needsXML = job->property("needsXML").toBool(); + + QList postList; + + if (needsXML) { + // Special cases for pools + postList = parseDanbooruResult(data, QString("pool"), &ok); + } else { + postList = parseDanbooruResult(data, &ok); + } if (!ok) { Q_EMIT(downloadError(QString("Unable to decode data"))); return; } - QList postList = result.toList(); - // How many posts do we have to fetch? m_currentPosts = postList.length(); - Q_FOREACH(QVariant element, postList) { + Q_FOREACH(const QVariant& element, postList) { QVariantMap map = element.toMap(); DanbooruPost* post = new DanbooruPost(map); +// qDebug() << "Got rating" << post->rating(); + + // First check, for rating + + if (post->rating() > m_maxRating) { + qDebug() << "Skipping " << post->fileUrl(); + qDebug() << "Rating was " << post->rating(); + m_currentPosts--; + delete post; + continue; + } + + // second check, blacklist + // We make a copy due to the fact that otherwise intersect() + // will change the set in place + + QSet temp = m_blacklist; + + QSet overlap = temp.intersect(post->tags()); + + if (!overlap.isEmpty()) { + + // Blacklisted tags are present, do not use this post + qDebug() << "Skipping " << post->fileUrl(); + qDebug() << "Blacklisted tags " << overlap; + m_currentPosts--; + delete post; + continue; + } + StoredTransferJob* pixmapJob = KIO::storedGet(post->thumbnailUrl(), KIO::NoReload, KIO::HideProgressInfo); @@ -238,6 +342,46 @@ namespace Danbooru { } + void DanbooruService::processTagList(KJob* job) + { + qDebug() << "Got tag data OK"; + + if (job->error()) { + qDebug() << "Something went wrong"; + Q_EMIT(downloadError(job->errorString())); + } + + StoredTransferJob* jobResult = qobject_cast(job); + + if (jobResult == 0) { + Q_EMIT(downloadError(QString("Internal error"))); + return; + } + + QByteArray data = jobResult->data(); + + bool ok; + + // Most Danbooru implementations return tags in wrong order when + // using JSON, so we have to fall back to XML + QList tagList = parseDanbooruResult(data, "tag", &ok); + + if (!ok) { + qDebug() << "Something went wrong"; + Q_EMIT(downloadError(QString("Unable to decode data"))); + return; + } + + qDebug() << "Populating tag list, length" << tagList.length(); + + Q_FOREACH(const QVariant& element, tagList) { + QVariantMap map = element.toMap(); + DanbooruTag* tag = new DanbooruTag(map); + Q_EMIT(tagDownloaded(tag)); + } + } + + void DanbooruService::processRelatedTagList(KJob* job) { Q_UNUSED(job) @@ -262,22 +406,16 @@ namespace Danbooru { QByteArray data = jobResult->data(); - QJson::Parser parser; - bool ok; - QVariant result = parser.parse(data, &ok); + QList poolList = parseDanbooruResult(data, &ok); if (!ok) { Q_EMIT(downloadError(QString("Unable to decode data"))); return; } - QList poolList = result.toList(); - - int currentPools = poolList.length(); - - Q_FOREACH(QVariant element, poolList) { + Q_FOREACH(const QVariant& element, poolList) { QVariantMap map = element.toMap(); DanbooruPool* pool = new DanbooruPool(map); @@ -335,6 +473,5 @@ namespace Danbooru { } - } // namespace Danbooru diff --git a/src/libdanbooru/danbooruservice.h b/src/libdanbooru/danbooruservice.h index e0d4306..4de0a0e 100644 --- a/src/libdanbooru/danbooruservice.h +++ b/src/libdanbooru/danbooruservice.h @@ -31,42 +31,32 @@ * **/ +// Qt + #include #include #include +#include + +// KDE #include #include +// Own + +#include "danbooru.h" + class QPixmap; class KUrl; class KJob; -/** - * @brief The Danbooru namespace. - * - **/ -namespace Danbooru { - /** - * @brief Types of tags - * - * A Danbooru tag is not simply a string, but carries some (limited) - * semantic information. In particular, tags are organized in what they - * refer to, either something related to the image itself, or to the - * artist that drew it, or the copyrights associated to the image, or even - * the characters that are represented in it. - * - **/ - enum TagType { - General, /**< Generic tags **/ - Artist, /**< Tags related to artists **/ - Copyright, /** RATINGS; - // member variables KUrl m_url; QString m_username; QString m_password; - QStringList m_blacklist; - QString m_maxRating; + QSet m_blacklist; + Ratings m_maxRating; unsigned int m_currentPosts; // To tell when to quit - // private functions - - /** - * @internal - * - **/ - static const QMap< QString, QStringList > initRatings(); - - public: /** @@ -169,10 +146,15 @@ namespace Danbooru { /** * @brief Get a list of tags from the board * + * If name is supplied, a list of tags including the exact name of the + * tag is fetched from Danbooru, otherwise the most recent tags are + * retrieved. + * * The tagDownloaded signal is emitted every time a tag has been * retrieved. * * @param limit The number of tags to get. + * @param name The name of the tag to retrieve, or an empty string * **/ void getTagList(int limit=10, QString name=""); @@ -184,7 +166,7 @@ namespace Danbooru { * @param tagType The type of tag to query for **/ void getRelatedTags(QStringList tags=QStringList(), - TagType tagType=General); + Danbooru::TagType tagType=General); /** * @return The currently allowed ratings when downloading posts. @@ -194,12 +176,14 @@ namespace Danbooru { /** * @return The maximum allowed rating for a post. **/ - const QString maximumAllowedRating() const; + const Ratings maximumAllowedRating() const; /** * @return The currently blacklisted tags. **/ - QStringList blacklist() const; + const QSet blacklist() const; + + void setBlacklist(const QSet& blacklist); /** * @brief Set the maximum allowed rating for the board. @@ -208,7 +192,7 @@ namespace Danbooru { * downloaded. * **/ - void setMaximumAllowedRating(QString rating); + void setMaximumAllowedRating(const Ratings& rating); /** * @brief Set the tag blacklist. @@ -221,6 +205,7 @@ namespace Danbooru { private Q_SLOTS: void processPostList(KJob* job); void processPoolList(KJob* job); + void processTagList(KJob* job); void processRelatedTagList(KJob* job); void downloadThumbnail(KJob* job); void downloadAllTags(KJob* job); @@ -258,6 +243,8 @@ namespace Danbooru { void postDownloaded(Danbooru::DanbooruPost* post); void poolDownloaded(Danbooru::DanbooruPool* pool); + + void tagDownloaded(Danbooru::DanbooruTag* tag); // TODO: Tags and similar };