/* * Copyright 2015 Luca Beltrame * * This file is part of Danbooru Client. * * Danbooru Client is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * Danbooru Client is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with Danbooru Client. If not, see . */ // STL #include // Qt #include "libdanbooru_debug.h" // KDE #include #include #include #include // Own #include "danbooruservice.h" #include "danboorupost.h" #include "danboorupool.h" #include "danboorutag.h" #include "utils.h" namespace Danbooru { using KIO::StoredTransferJob; using KIO::MultiGetJob; const QString DanbooruService::POST_URL = "post/index.json" ; const QString DanbooruService::TAG_URL = "tag/index.xml"; const QString DanbooruService::POOL_URL = "pool/index.json"; const QString DanbooruService::ARTIST_URL = "artist/index.json"; const QString DanbooruService::POOL_DATA_URL = "pool/show.xml"; const QString DanbooruService::RELATED_TAG_URL = "tag/related.json"; DanbooruService::DanbooruService(QUrl boardUrl, QString username, QString password, KImageCache *cache, QObject *parent): QObject(parent), m_url(boardUrl), m_username(username), m_password(password), m_maxRating(Danbooru::DanbooruPost::Safe), m_maxPosts(10), m_currentPage(1), m_tags(QStringList()), m_postsToFetch(0), m_cache(cache) { } DanbooruService::~DanbooruService() { } void DanbooruService::getPostList() { // We can't fetch more than 100 items, API limitation QMap parameters; parameters.insert("limit", QString::number(m_maxPosts)); parameters.insert("page", QString::number(m_currentPage)); QUrl danbooruUrl = requestUrl(m_url, POST_URL, m_username, m_password, parameters, m_tags); // qCDebug(LIBDANBOORU) << "Final constructed post URL" << danbooruUrl; KIO::StoredTransferJob *job = KIO::storedGet(danbooruUrl, KIO::NoReload, KIO::HideProgressInfo); // This job can use JSON data job->setProperty("needsXML", false); connect(job, &KIO::StoredTransferJob::result, this, &DanbooruService::processPostList); } void DanbooruService::getTagList(int limit, QString name) { QMap parameters; parameters.insert("limit", QString::number(limit)); if (!name.isEmpty()) { parameters.insert("name", name); } parameters.insert("order", "date"); QUrl danbooruUrl = requestUrl(m_url, TAG_URL, m_username, m_password, parameters); // qCDebug(LIBDANBOORU) << "Final constructed tag URL" << danbooruUrl.url(); KIO::StoredTransferJob *job = KIO::storedGet(danbooruUrl, KIO::NoReload, KIO::HideProgressInfo); connect(job, &KIO::StoredTransferJob::result, this, &DanbooruService::processTagList); } void DanbooruService::getPool(int poolId, int page) { QMap parameters; parameters.insert("id", QString::number(poolId)); if (page > 1) { parameters.insert("page", QString::number(page)); } QUrl danbooruUrl = requestUrl(m_url, POOL_DATA_URL, m_username, m_password, parameters); // qCDebug(LIBDANBOORU) << "Final constructed pool URL" << danbooruUrl.url(); KIO::StoredTransferJob *job = KIO::storedGet(danbooruUrl, KIO::NoReload, KIO::HideProgressInfo); //HACK: Most Danbooru implementations don't provide valid data on // pools via JSON, hence we set XML and parse the XML data. job->setProperty("needsXML", true); connect(job, &KIO::StoredTransferJob::result, this, &DanbooruService::processPostList); } void DanbooruService::getPoolList() { QUrl danbooruUrl; if (m_currentPage == 1) { danbooruUrl = requestUrl(m_url, POOL_URL, m_username, m_password); } else { QMap map; map.insert("page", QString::number(m_currentPage)); danbooruUrl = requestUrl(m_url, POOL_URL, m_username, m_password, map); } // qCDebug(LIBDANBOORU) << "Final constructed pool list URL" << danbooruUrl.url(); KIO::StoredTransferJob *job = KIO::storedGet(danbooruUrl, KIO::NoReload, KIO::HideProgressInfo); // This job can use JSON data job->setProperty("needsXML", false); // connect(job, &KIO::StoredTransferJob::result, this, &DanbooruService::processPoolList); connect(job, &KIO::StoredTransferJob::result, [this](KJob * job) { if (job->error()) { Q_EMIT(downloadError(job->errorString())); return; } StoredTransferJob *jobResult = qobject_cast(job); QByteArray data = jobResult->data(); bool ok; QList poolList = parseDanbooruResult(data, &ok).toList(); if (!ok) { Q_EMIT(downloadError(QString("Unable to decode data"))); return; } for (auto element : poolList) { QVariantMap map = element.toMap(); DanbooruPool *pool = new DanbooruPool(map); Q_EMIT(poolDownloaded(pool)); } //qCDebug(LIBDANBOORU) << "Pool download finished!"; Q_EMIT(poolDownloadFinished()); } ); } void DanbooruService::getRelatedTags(const QStringList &tags, DanbooruTag::TagType tagType) { QString type; switch (tagType) { case DanbooruTag::General: type = "general"; break; case DanbooruTag::Artist: type = "artist"; break; case DanbooruTag::Copyright: type = "copyright"; break; case DanbooruTag::Character: type = "character"; break; case DanbooruTag::Unknown: type = "unknown"; break; } QMap parameters; parameters.insert("type", type); QUrl danbooruUrl = requestUrl(m_url, RELATED_TAG_URL, m_username, m_password, parameters, tags); // qCDebug(LIBDANBOORU) << "Final constructed related tag URL" << danbooruUrl; StoredTransferJob *job = KIO::storedGet( danbooruUrl, KIO::NoReload, KIO::HideProgressInfo ); connect(job, &StoredTransferJob::result, [this](KJob * job) { if (job->error()) { Q_EMIT(downloadError(job->errorString())); return; } StoredTransferJob *jobResult = qobject_cast(job); QByteArray data = jobResult->data(); bool ok; QVariantMap tagList = parseDanbooruResult(data, &ok).toMap(); if (!ok) { Q_EMIT(downloadError(QString("Unable to decode data"))); return; } QVariantMap::const_iterator iter; // The service returns a list of key-related tag list pair, // we iterate through them and remove the empty (not found) ones, then // we call getTagList. Unfortunately Danbooru doesn't have a method to // fetch all tags in batch, so this is done one by one. for (iter = tagList.constBegin(); iter != tagList.constEnd(); ++iter) { QList tags = iter.value().toList(); if (tags.isEmpty()) { continue; } for (auto tag : tags) { // We get the first element in the list, the second is // the ID which is useless (no API methods in Danbooru) QString tagName = tag.toList()[0].toString(); getTagList(1, tagName); } } } ); } const QSet< QString > DanbooruService::blacklist() const { return m_blacklist; } const QStringList DanbooruService::allowedRatings() const { QStringList ratings; if (m_maxRating.testFlag(DanbooruPost::Safe)) { ratings.append("Safe"); } if (m_maxRating.testFlag(DanbooruPost::Questionable)) { ratings.append("Questionable"); } if (m_maxRating.testFlag(DanbooruPost::Explicit)) { ratings.append("Explicit"); } return ratings; } int DanbooruService::currentPage() const { return m_currentPage; } const DanbooruPost::Ratings DanbooruService::maximumAllowedRating() const { return m_maxRating; } int DanbooruService::maxPosts() const { return m_maxPosts; } void DanbooruService::nextPostPage() { m_currentPage++; getPostList(); } void DanbooruService::nextPoolPage() { m_currentPage++; getPoolList(); } QStringList DanbooruService::postTags() const { return m_tags; } void DanbooruService::reset() { m_currentPage = 1; m_tags = QStringList(); } // Slots void DanbooruService::processTagList(KJob *job) { if (job->error()) { Q_EMIT(downloadError(job->errorString())); return; } StoredTransferJob *jobResult = qobject_cast(job); QByteArray data = jobResult->data(); bool ok; // Most Danbooru implementations return tags in wrong order when // using JSON, so we have to fall back to XML QList tagList = parseDanbooruResult(data, "tag", &ok); if (!ok) { Q_EMIT(downloadError(QString("Unable to decode data"))); return; } for (auto element : tagList) { QVariantMap map = element.toMap(); DanbooruTag *tag = new DanbooruTag(map); if (!tag) { continue; } Q_EMIT(tagDownloaded(tag)); } } void DanbooruService::processPostList(KJob *job) { // qCDebug(LIBDANBOORU) << "Got post data OK"; if (job->error()) { Q_EMIT(downloadError(job->errorString())); } StoredTransferJob *jobResult = qobject_cast(job); if (jobResult == 0) { Q_EMIT(downloadError(QString("Internal error"))); return; } QByteArray data = jobResult->data(); bool ok; bool needsXML = job->property("needsXML").toBool(); QList postList; if (needsXML) { // Special cases for pools postList = parseDanbooruResult(data, QString("post"), &ok); } else { postList = parseDanbooruResult(data, &ok).toList(); } if (!ok) { Q_EMIT(downloadError(QString("Unable to decode data"))); return; } // How many posts do we have to fetch? m_postsToFetch = postList.length(); for (auto element : postList) { QVariantMap map = element.toMap(); DanbooruPost *post = new DanbooruPost(map); // Remove unwanted posts if (isPostBlacklisted(post, m_blacklist, m_maxRating)) { m_postsToFetch--; delete post; continue; } QPixmap pix; // qCDebug(LIBDANBOORU) << "About to donwload images"; if (m_cache && m_cache->findPixmap(post->thumbnailUrl().url(), &pix)) { post->setPixmap(pix); Q_EMIT(postDownloaded(post)); m_postsToFetch--; if (m_postsToFetch == 0) { qCDebug(LIBDANBOORU) << "Post download finished"; Q_EMIT(postDownloadFinished()); } } else { StoredTransferJob *pixmapJob = KIO::storedGet(post->thumbnailUrl(), KIO::NoReload, KIO::HideProgressInfo); // We don't want to overload the servers, so set some rational // priority KIO::Scheduler::setJobPriority(static_cast(job), 1); QVariant variant; variant.setValue(post); // pixmapJob->setProperty("danbooruPost", variant); connect(pixmapJob, &StoredTransferJob::result, [post, this, pix](KJob * job) mutable { if (job->error()) { Q_EMIT(downloadError(job->errorString())); return; } StoredTransferJob *jobResult = qobject_cast(job); if (!pix.loadFromData(jobResult->data())) { Q_EMIT(downloadError(QString("Pixmap data could not be loaded"))); return; } post->setPixmap(pix); if (m_cache) { //qCDebug(LIBDANBOORU) << "Inserting item in cache"; m_cache->insertPixmap(post->thumbnailUrl().url(), pix); } m_postsToFetch--; // One less post to do //qCDebug(LIBDANBOORU) << "Current posts remaining" << m_currentPosts; Q_EMIT(postDownloaded(post)); if (m_postsToFetch == 0) { qCDebug(LIBDANBOORU) << "Post download finished"; Q_EMIT(postDownloadFinished()); } }); } } } void DanbooruService::downloadAllTags(KJob *job) { Q_UNUSED(job) } void DanbooruService::setBlacklist(const QSet< QString > &blacklist) { if (!blacklist.isEmpty()) { m_blacklist = blacklist; } } void DanbooruService::setBlacklist(const QStringList &blacklist) { if (blacklist.isEmpty()) { return; } m_blacklist.clear(); for (auto element : blacklist) { m_blacklist.insert(element); } } void DanbooruService::setBoardUrl(const QUrl &url) { m_url = url; } void DanbooruService::setMaximumAllowedRating(DanbooruPost::Rating rating) { DanbooruPost::Ratings flags; switch (rating) { case DanbooruPost::Safe: flags = DanbooruPost::Safe; break; case DanbooruPost::Questionable: flags = DanbooruPost::Safe | DanbooruPost::Questionable; break; case DanbooruPost::Explicit: flags = DanbooruPost::Safe | DanbooruPost::Questionable | DanbooruPost::Explicit; break; } m_maxRating = flags; } void DanbooruService::setUserName(const QString &username) { if (username.isEmpty()) { return; } m_username = username; } void DanbooruService::setPassword(const QString &password) { if (password.isEmpty()) { return; } m_password = password; } void DanbooruService::setPostTags(const QStringList &tags) { if (!tags.isEmpty()) { m_tags = tags; } } void DanbooruService::setImageCache(KImageCache *cache) { m_cache = cache; } void DanbooruService::setMaxPosts(int number) { m_maxPosts = number < 100 ? number : 100; } void DanbooruService::setCurrentPage(int page) { m_currentPage = page; } } // namespace Danbooru