Implement most missing functions

This commit is contained in:
Luca Beltrame 2013-03-17 12:00:30 +01:00
parent 816b370302
commit 869620d6f6
2 changed files with 217 additions and 93 deletions

View file

@ -20,30 +20,34 @@
* *
*/ */
// Qt
#include <QtCore/QDebug> #include <QtCore/QDebug>
#include <qjson/parser.h> #include <qjson/parser.h>
// KDE
#include <KIO/Job> #include <KIO/Job>
#include <KIO/Scheduler> #include <KIO/Scheduler>
#include "utils.h" // Own
#include "danbooruservice.h" #include "danbooruservice.h"
#include "danboorupost.h" #include "danboorupost.h"
#include "danboorupool.h" #include "danboorupool.h"
#include "danboorutag.h"
#include "utils.h"
namespace Danbooru { namespace Danbooru {
using KIO::StoredTransferJob; using KIO::StoredTransferJob;
const QString DanbooruService::POST_URL ="post/index.json" ; const QString DanbooruService::POST_URL ="post/index.json" ;
const QString DanbooruService::TAG_URL = "tag/index.json"; const QString DanbooruService::TAG_URL = "tag/index.xml";
const QString DanbooruService::POOL_URL = "pool/index.json"; const QString DanbooruService::POOL_URL = "pool/index.json";
const QString DanbooruService::ARTIST_URL = "artist/index.json"; const QString DanbooruService::ARTIST_URL = "artist/index.json";
const QString DanbooruService::POOL_DATA_URL = "pool/show.xml"; const QString DanbooruService::POOL_DATA_URL = "pool/show.xml";
const QString DanbooruService::RELATED_TAG_URL = "tag/related.json"; const QString DanbooruService::RELATED_TAG_URL = "tag/related.json";
const QMap<QString, QStringList> DanbooruService::RATINGS = initRatings();
DanbooruService::DanbooruService(KUrl& boardUrl, QString username, DanbooruService::DanbooruService(KUrl& boardUrl, QString username,
QString password, QString password,
@ -52,7 +56,7 @@ namespace Danbooru {
m_url(boardUrl), m_url(boardUrl),
m_username(username), m_username(username),
m_password(password), m_password(password),
m_maxRating("Safe"), m_maxRating(Safe),
m_currentPosts(0) m_currentPosts(0)
{ {
} }
@ -62,26 +66,6 @@ namespace Danbooru {
} }
const QMap< QString, QStringList > DanbooruService::initRatings()
{
QMap<QString, QStringList> map;
QStringList safeRatings;
safeRatings.append(QString("Safe"));
QStringList questionableRatings = QStringList(safeRatings);
questionableRatings.append(QString("Questionable"));
QStringList explicitRatings = QStringList(questionableRatings);
explicitRatings.append(QString("Explicit"));
map.insert("Safe", safeRatings);
map.insert("Questionable", questionableRatings);
map.insert("Explicit", explicitRatings);
return map;
}
void DanbooruService::getPostList(int page, QStringList tags, int limit) void DanbooruService::getPostList(int page, QStringList tags, int limit)
{ {
@ -90,19 +74,22 @@ namespace Danbooru {
limit = limit > 100 ? 100: limit; limit = limit > 100 ? 100: limit;
QMap<QString, QString> map; QMap<QString, QString> parameters;
map.insert("limit", QString::number(limit)); parameters.insert("limit", QString::number(limit));
map.insert("page", QString::number(page)); parameters.insert("page", QString::number(page));
KUrl danbooruUrl = requestUrl(m_url, POST_URL, m_username, KUrl danbooruUrl = requestUrl(m_url, POST_URL, m_username,
m_password, map, tags); m_password, parameters, tags);
qDebug() << "Final constructed URL" << danbooruUrl.url(); qDebug() << "Final constructed URL" << danbooruUrl.url();
KIO::StoredTransferJob* job = KIO::storedGet(danbooruUrl, KIO::NoReload, KIO::StoredTransferJob* job = KIO::storedGet(danbooruUrl, KIO::NoReload,
KIO::HideProgressInfo); KIO::HideProgressInfo);
// This job can use JSON data
job->setProperty("needsXML", false);
connect(job, SIGNAL(result(KJob*)), this, connect(job, SIGNAL(result(KJob*)), this,
SLOT(processPostList(KJob*))); SLOT(processPostList(KJob*)));
@ -110,12 +97,51 @@ namespace Danbooru {
void DanbooruService::getTagList(int limit, QString name) void DanbooruService::getTagList(int limit, QString name)
{ {
QMap<QString, QString> parameters;
parameters.insert("limit", QString::number(limit));
if (!name.isEmpty()) {
parameters.insert("name", name);
}
KUrl danbooruUrl = requestUrl(m_url, TAG_URL, m_username, m_password,
parameters);
qDebug() << "Final constructed URL" << danbooruUrl.url();
KIO::StoredTransferJob* job = KIO::storedGet(danbooruUrl, KIO::NoReload,
KIO::HideProgressInfo);
connect(job, SIGNAL(result(KJob*)), this, SLOT(processTagList(KJob*)));
} }
void DanbooruService::getPool(int poolId, int page) void DanbooruService::getPool(int poolId, int page)
{ {
QMap<QString, QString> parameters;
parameters.insert("id", QString::number(poolId));
if (page > 1) {
parameters.insert("page", QString::number(page));
}
KUrl danbooruUrl = requestUrl(m_url, POOL_DATA_URL, m_username,
m_password, parameters);
qDebug() << "Final constructed URL" << danbooruUrl.url();
KIO::StoredTransferJob* job = KIO::storedGet(danbooruUrl, KIO::NoReload,
KIO::HideProgressInfo);
//HACK: Most Danbooru implementations don't provide valid data on
// pools via JSON, hence we set XML and parse the XML data.
job->setProperty("needsXML", true);
connect(job, SIGNAL(result(KJob*)), this,
SLOT(processPostList(KJob*)));
} }
void DanbooruService::getPoolList(int page) void DanbooruService::getPoolList(int page)
@ -137,40 +163,81 @@ namespace Danbooru {
KIO::StoredTransferJob* job = KIO::storedGet(danbooruUrl, KIO::NoReload, KIO::StoredTransferJob* job = KIO::storedGet(danbooruUrl, KIO::NoReload,
KIO::HideProgressInfo); KIO::HideProgressInfo);
// This job can use JSON data
job->setProperty("needsXML", false);
connect(job, SIGNAL(result(KJob*)), this, connect(job, SIGNAL(result(KJob*)), this,
SLOT(processPoolList(KJob*))); SLOT(processPoolList(KJob*)));
} }
void DanbooruService::getRelatedTags(QStringList tags, void DanbooruService::getRelatedTags(QStringList tags, Danbooru::TagType tagType)
Danbooru::TagType tagType)
{ {
} }
// Getters / setters // Getters / setters
QStringList DanbooruService::blacklist() const void DanbooruService::setBlacklist(const QSet< QString >& blacklist)
{
if (!blacklist.isEmpty()) {
m_blacklist = blacklist;
}
}
const QSet< QString > DanbooruService::blacklist() const
{ {
return m_blacklist; return m_blacklist;
} }
const QStringList DanbooruService::allowedRatings() const const QStringList DanbooruService::allowedRatings() const
{ {
return RATINGS.value(m_maxRating); QStringList ratings;
}
void DanbooruService::setMaximumAllowedRating(QString rating) if (m_maxRating.testFlag(Safe)) {
{ ratings.append("Safe");
if (RATINGS.contains(rating)) {
m_maxRating = rating;
} }
if (m_maxRating.testFlag(Questionable)) {
ratings.append("Questionable");
}
if (m_maxRating.testFlag(Explicit)) {
ratings.append("Explicit");
}
return ratings;
} }
const QString DanbooruService::maximumAllowedRating() const void DanbooruService::setMaximumAllowedRating(const Danbooru::Ratings& rating)
{ {
Ratings flags;
switch (rating) {
case Safe:
flags = Safe;
break;
case Questionable:
flags = Safe | Questionable;
break;
case Explicit:
flags = Safe | Questionable | Explicit;
break;
}
m_maxRating = flags;
}
const Ratings DanbooruService::maximumAllowedRating() const
{
return m_maxRating; return m_maxRating;
} }
// Slots // Slots
@ -194,29 +261,66 @@ namespace Danbooru {
QByteArray data = jobResult->data(); QByteArray data = jobResult->data();
QJson::Parser parser; qDebug() << jobResult->mimetype();
bool ok; bool ok;
QVariant result = parser.parse(data, &ok); bool needsXML = job->property("needsXML").toBool();
QList<QVariant> postList;
if (needsXML) {
// Special cases for pools
postList = parseDanbooruResult(data, QString("pool"), &ok);
} else {
postList = parseDanbooruResult(data, &ok);
}
if (!ok) { if (!ok) {
Q_EMIT(downloadError(QString("Unable to decode data"))); Q_EMIT(downloadError(QString("Unable to decode data")));
return; return;
} }
QList<QVariant> postList = result.toList();
// How many posts do we have to fetch? // How many posts do we have to fetch?
m_currentPosts = postList.length(); m_currentPosts = postList.length();
Q_FOREACH(QVariant element, postList) { Q_FOREACH(const QVariant& element, postList) {
QVariantMap map = element.toMap(); QVariantMap map = element.toMap();
DanbooruPost* post = new DanbooruPost(map); DanbooruPost* post = new DanbooruPost(map);
// qDebug() << "Got rating" << post->rating();
// First check, for rating
if (post->rating() > m_maxRating) {
qDebug() << "Skipping " << post->fileUrl();
qDebug() << "Rating was " << post->rating();
m_currentPosts--;
delete post;
continue;
}
// second check, blacklist
// We make a copy due to the fact that otherwise intersect()
// will change the set in place
QSet<QString> temp = m_blacklist;
QSet<QString> overlap = temp.intersect(post->tags());
if (!overlap.isEmpty()) {
// Blacklisted tags are present, do not use this post
qDebug() << "Skipping " << post->fileUrl();
qDebug() << "Blacklisted tags " << overlap;
m_currentPosts--;
delete post;
continue;
}
StoredTransferJob* pixmapJob = KIO::storedGet(post->thumbnailUrl(), StoredTransferJob* pixmapJob = KIO::storedGet(post->thumbnailUrl(),
KIO::NoReload, KIO::HideProgressInfo); KIO::NoReload, KIO::HideProgressInfo);
@ -238,6 +342,46 @@ namespace Danbooru {
} }
void DanbooruService::processTagList(KJob* job)
{
qDebug() << "Got tag data OK";
if (job->error()) {
qDebug() << "Something went wrong";
Q_EMIT(downloadError(job->errorString()));
}
StoredTransferJob* jobResult = qobject_cast<StoredTransferJob*>(job);
if (jobResult == 0) {
Q_EMIT(downloadError(QString("Internal error")));
return;
}
QByteArray data = jobResult->data();
bool ok;
// Most Danbooru implementations return tags in wrong order when
// using JSON, so we have to fall back to XML
QList<QVariant> tagList = parseDanbooruResult(data, "tag", &ok);
if (!ok) {
qDebug() << "Something went wrong";
Q_EMIT(downloadError(QString("Unable to decode data")));
return;
}
qDebug() << "Populating tag list, length" << tagList.length();
Q_FOREACH(const QVariant& element, tagList) {
QVariantMap map = element.toMap();
DanbooruTag* tag = new DanbooruTag(map);
Q_EMIT(tagDownloaded(tag));
}
}
void DanbooruService::processRelatedTagList(KJob* job) void DanbooruService::processRelatedTagList(KJob* job)
{ {
Q_UNUSED(job) Q_UNUSED(job)
@ -262,22 +406,16 @@ namespace Danbooru {
QByteArray data = jobResult->data(); QByteArray data = jobResult->data();
QJson::Parser parser;
bool ok; bool ok;
QVariant result = parser.parse(data, &ok); QList<QVariant> poolList = parseDanbooruResult(data, &ok);
if (!ok) { if (!ok) {
Q_EMIT(downloadError(QString("Unable to decode data"))); Q_EMIT(downloadError(QString("Unable to decode data")));
return; return;
} }
QList<QVariant> poolList = result.toList(); Q_FOREACH(const QVariant& element, poolList) {
int currentPools = poolList.length();
Q_FOREACH(QVariant element, poolList) {
QVariantMap map = element.toMap(); QVariantMap map = element.toMap();
DanbooruPool* pool = new DanbooruPool(map); DanbooruPool* pool = new DanbooruPool(map);
@ -335,6 +473,5 @@ namespace Danbooru {
} }
} // namespace Danbooru } // namespace Danbooru

View file

@ -31,42 +31,32 @@
* *
**/ **/
// Qt
#include <QtCore/QObject> #include <QtCore/QObject>
#include <QtCore/QString> #include <QtCore/QString>
#include <QtCore/QStringList> #include <QtCore/QStringList>
#include <QtCore/QSet>
// KDE
#include <KUrl> #include <KUrl>
#include <KIO/JobClasses> #include <KIO/JobClasses>
// Own
#include "danbooru.h"
class QPixmap; class QPixmap;
class KUrl; class KUrl;
class KJob; class KJob;
/**
* @brief The Danbooru namespace.
*
**/
namespace Danbooru {
/** namespace Danbooru {
* @brief Types of tags
*
* A Danbooru tag is not simply a string, but carries some (limited)
* semantic information. In particular, tags are organized in what they
* refer to, either something related to the image itself, or to the
* artist that drew it, or the copyrights associated to the image, or even
* the characters that are represented in it.
*
**/
enum TagType {
General, /**< Generic tags **/
Artist, /**< Tags related to artists **/
Copyright, /**<Tags related to copyrights **/
Character /**<Tags related to characters **/
};
class DanbooruPost; class DanbooruPost;
class DanbooruPool; class DanbooruPool;
class DanbooruTag;
using KIO::StoredTransferJob; using KIO::StoredTransferJob;
@ -96,29 +86,16 @@ namespace Danbooru {
static const QString POOL_DATA_URL; static const QString POOL_DATA_URL;
static const QString RELATED_TAG_URL; static const QString RELATED_TAG_URL;
// Ratings
static const QMap<QString, QStringList> RATINGS;
// member variables // member variables
KUrl m_url; KUrl m_url;
QString m_username; QString m_username;
QString m_password; QString m_password;
QStringList m_blacklist; QSet<QString> m_blacklist;
QString m_maxRating; Ratings m_maxRating;
unsigned int m_currentPosts; // To tell when to quit unsigned int m_currentPosts; // To tell when to quit
// private functions
/**
* @internal
*
**/
static const QMap< QString, QStringList > initRatings();
public: public:
/** /**
@ -169,10 +146,15 @@ namespace Danbooru {
/** /**
* @brief Get a list of tags from the board * @brief Get a list of tags from the board
* *
* If name is supplied, a list of tags including the exact name of the
* tag is fetched from Danbooru, otherwise the most recent tags are
* retrieved.
*
* The tagDownloaded signal is emitted every time a tag has been * The tagDownloaded signal is emitted every time a tag has been
* retrieved. * retrieved.
* *
* @param limit The number of tags to get. * @param limit The number of tags to get.
* @param name The name of the tag to retrieve, or an empty string
* *
**/ **/
void getTagList(int limit=10, QString name=""); void getTagList(int limit=10, QString name="");
@ -184,7 +166,7 @@ namespace Danbooru {
* @param tagType The type of tag to query for * @param tagType The type of tag to query for
**/ **/
void getRelatedTags(QStringList tags=QStringList(), void getRelatedTags(QStringList tags=QStringList(),
TagType tagType=General); Danbooru::TagType tagType=General);
/** /**
* @return The currently allowed ratings when downloading posts. * @return The currently allowed ratings when downloading posts.
@ -194,12 +176,14 @@ namespace Danbooru {
/** /**
* @return The maximum allowed rating for a post. * @return The maximum allowed rating for a post.
**/ **/
const QString maximumAllowedRating() const; const Ratings maximumAllowedRating() const;
/** /**
* @return The currently blacklisted tags. * @return The currently blacklisted tags.
**/ **/
QStringList blacklist() const; const QSet<QString> blacklist() const;
void setBlacklist(const QSet<QString>& blacklist);
/** /**
* @brief Set the maximum allowed rating for the board. * @brief Set the maximum allowed rating for the board.
@ -208,7 +192,7 @@ namespace Danbooru {
* downloaded. * downloaded.
* *
**/ **/
void setMaximumAllowedRating(QString rating); void setMaximumAllowedRating(const Ratings& rating);
/** /**
* @brief Set the tag blacklist. * @brief Set the tag blacklist.
@ -221,6 +205,7 @@ namespace Danbooru {
private Q_SLOTS: private Q_SLOTS:
void processPostList(KJob* job); void processPostList(KJob* job);
void processPoolList(KJob* job); void processPoolList(KJob* job);
void processTagList(KJob* job);
void processRelatedTagList(KJob* job); void processRelatedTagList(KJob* job);
void downloadThumbnail(KJob* job); void downloadThumbnail(KJob* job);
void downloadAllTags(KJob* job); void downloadAllTags(KJob* job);
@ -258,6 +243,8 @@ namespace Danbooru {
void postDownloaded(Danbooru::DanbooruPost* post); void postDownloaded(Danbooru::DanbooruPost* post);
void poolDownloaded(Danbooru::DanbooruPool* pool); void poolDownloaded(Danbooru::DanbooruPool* pool);
void tagDownloaded(Danbooru::DanbooruTag* tag);
// TODO: Tags and similar // TODO: Tags and similar
}; };