danbooru-client/src/libdanbooru/konachan.cpp
2015-08-26 16:23:25 +02:00

486 lines
13 KiB
C++

/*
* Copyright 2015 Luca Beltrame <lbeltrame@kde.org>
*
* This file is part of Danbooru Client.
*
* Danbooru Client is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Danbooru Client is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Danbooru Client. If not, see <http://www.gnu.org/licenses/>.
*/
#include "konachan.h"
#include "danboorupost.h"
#include "danboorupool.h"
#include "danboorutag.h"
#include "utils.h"
#include "libdanbooru_debug.h"
// KF5
#include <KIO/Job>
#include <KIO/Scheduler>
#include <KImageCache>
using KIO::StoredTransferJob;
namespace Danbooru {
///////
// URIs
///////
const QLatin1String KonachanDanbooruService::postUri() const {
return QLatin1String("post/index.json");
}
const QLatin1String KonachanDanbooruService::poolUri() const {
return QLatin1String("pool/index.json");
}
const QLatin1String KonachanDanbooruService::artistUri() const {
return QLatin1String("artist/index.json");
}
const QLatin1String KonachanDanbooruService::tagUri() const {
return QLatin1String("tag/index.xml");
}
const QLatin1String KonachanDanbooruService::poolDataUri() const {
return QLatin1String("pool/show.xml");
}
const QLatin1String KonachanDanbooruService::relatedTagUri() const {
return QLatin1String("tag/related.json");
}
////////////////
// Other methods
////////////////
DanbooruServiceBase::SupportedOperations KonachanDanbooruService::supportedOperations() const {
return SupportedOperations(DanbooruServiceBase::PostDownload | DanbooruServiceBase::PoolDownload |
DanbooruServiceBase::TagDownload | DanbooruServiceBase::TagSearch |
DanbooruServiceBase::RelatedTagSearch);
}
Danbooru::ApiType KonachanDanbooruService::apiType() const {
return Danbooru::ApiType::KonachanDanbooru;
}
KonachanDanbooruService::~KonachanDanbooruService() {
}
//////////////
// API methods
//////////////
void KonachanDanbooruService::getPostList() {
// We can't fetch more than 100 items, API limitation
QMap<QString, QString> parameters;
parameters.insert("limit", QString::number(m_maxPosts));
parameters.insert("page", QString::number(m_currentPage));
QUrl danbooruUrl = requestUrl(m_url, postUri(), m_username,
m_password, parameters, m_tags);
qCDebug(LIBDANBOORU) << "Final constructed post URL" << danbooruUrl;
KIO::StoredTransferJob *job = KIO::storedGet(danbooruUrl, KIO::NoReload,
KIO::HideProgressInfo);
// This job can use JSON data
job->setProperty("needsXML", false);
connect(job, &KIO::StoredTransferJob::result, this, &KonachanDanbooruService::processPostList);
}
void KonachanDanbooruService::getPoolList(int limit)
{
QUrl danbooruUrl;
if (m_currentPage == 1) {
danbooruUrl = requestUrl(m_url, poolUri(), m_username, m_password);
} else {
QMap<QString, QString> map;
map.insert("page", QString::number(m_currentPage));
danbooruUrl = requestUrl(m_url, poolUri(), m_username,
m_password, map);
}
// qCDebug(LIBDANBOORU) << "Final constructed pool list URL" << danbooruUrl.url();
KIO::StoredTransferJob *job = KIO::storedGet(danbooruUrl, KIO::NoReload,
KIO::HideProgressInfo);
// This job can use JSON data
job->setProperty("needsXML", false);
// connect(job, &KIO::StoredTransferJob::result, this, &DanbooruService::processPoolList);
connect(job, &KIO::StoredTransferJob::result, [this, limit](KJob * job) {
if (job->error()) {
Q_EMIT(downloadError(job->errorString()));
return;
}
StoredTransferJob *jobResult = qobject_cast<StoredTransferJob *>(job);
QByteArray data = jobResult->data();
bool ok;
QList<QVariant> poolList = parseDanbooruResult(data, &ok).toList();
if (!ok) {
Q_EMIT(downloadError(QString("Unable to decode data")));
return;
}
if (limit > 0 && poolList.length() > limit) {
poolList = poolList.mid(0, limit);
}
for (auto element : poolList) {
QVariantMap map = element.toMap();
DanbooruPool *pool = new DanbooruPool(map);
Q_EMIT(poolDownloaded(pool));
}
//qCDebug(LIBDANBOORU) << "Pool download finished!";
Q_EMIT(poolDownloadFinished());
}
);
}
void KonachanDanbooruService::getPool(int poolId, int page)
{
QMap<QString, QString> parameters;
parameters.insert("id", QString::number(poolId));
if (page > 1) {
parameters.insert("page", QString::number(page));
}
QUrl danbooruUrl = requestUrl(m_url, poolDataUri(), m_username,
m_password, parameters);
// qCDebug(LIBDANBOORU) << "Final constructed pool URL" << danbooruUrl.url();
KIO::StoredTransferJob *job = KIO::storedGet(danbooruUrl, KIO::NoReload,
KIO::HideProgressInfo);
//HACK: Most Danbooru implementations don't provide valid data on
// pools via JSON, hence we set XML and parse the XML data.
job->setProperty("needsXML", true);
connect(job, &KIO::StoredTransferJob::result, this, &KonachanDanbooruService::processPostList);
}
void KonachanDanbooruService::getTagList(int limit, QString name)
{
QMap<QString, QString> parameters;
parameters.insert("limit", QString::number(limit));
if (!name.isEmpty()) {
parameters.insert("name", name);
}
parameters.insert("order", "date");
QUrl danbooruUrl = requestUrl(m_url, tagUri(), m_username, m_password,
parameters);
// qCDebug(LIBDANBOORU) << "Final constructed tag URL" << danbooruUrl.url();
KIO::StoredTransferJob *job = KIO::storedGet(danbooruUrl, KIO::NoReload,
KIO::HideProgressInfo);
connect(job, &KIO::StoredTransferJob::result, this, &KonachanDanbooruService::processTagList);
}
void KonachanDanbooruService::getRelatedTags(const QStringList &tags,
DanbooruTag::TagType tagType)
{
QString type;
switch (tagType) {
case DanbooruTag::General:
type = "general";
break;
case DanbooruTag::Artist:
type = "artist";
break;
case DanbooruTag::Copyright:
type = "copyright";
break;
case DanbooruTag::Character:
type = "character";
break;
case DanbooruTag::Unknown:
type = "unknown";
break;
}
QMap<QString, QString> parameters;
parameters.insert("type", type);
QUrl danbooruUrl = requestUrl(m_url, relatedTagUri(), m_username,
m_password, parameters, tags);
// qCDebug(LIBDANBOORU) << "Final constructed related tag URL" << danbooruUrl;
StoredTransferJob *job = KIO::storedGet(
danbooruUrl, KIO::NoReload,
KIO::HideProgressInfo
);
connect(job, &StoredTransferJob::result, [this](KJob * job) {
if (job->error()) {
Q_EMIT(downloadError(job->errorString()));
return;
}
StoredTransferJob *jobResult = qobject_cast<StoredTransferJob *>(job);
QByteArray data = jobResult->data();
bool ok;
QVariantMap tagList = parseDanbooruResult(data, &ok).toMap();
if (!ok) {
Q_EMIT(downloadError(QString("Unable to decode data")));
return;
}
QVariantMap::const_iterator iter;
// The service returns a list of key-related tag list pair,
// we iterate through them and remove the empty (not found) ones, then
// we call getTagList. Unfortunately Danbooru doesn't have a method to
// fetch all tags in batch, so this is done one by one.
for (iter = tagList.constBegin(); iter != tagList.constEnd(); ++iter) {
QList<QVariant> tags = iter.value().toList();
if (tags.isEmpty()) {
continue;
}
for (auto tag : tags) {
// We get the first element in the list, the second is
// the ID which is useless (no API methods in Danbooru)
QString tagName = tag.toList().at(0).toString();
getTagList(1, tagName);
}
}
}
);
}
////////
// Slots
////////
void KonachanDanbooruService::processTagList(KJob *job)
{
if (job->error()) {
Q_EMIT(downloadError(job->errorString()));
return;
}
StoredTransferJob *jobResult = qobject_cast<StoredTransferJob *>(job);
QByteArray data = jobResult->data();
bool ok;
// Most Danbooru implementations return tags in wrong order when
// using JSON, so we have to fall back to XML
QList<QVariant> tagList = parseDanbooruResult(data, "tag", &ok);
if (!ok) {
Q_EMIT(downloadError(QString("Unable to decode data")));
return;
}
for (auto element : tagList) {
QVariantMap map = element.toMap();
DanbooruTag *tag = new DanbooruTag(map);
if (!tag) {
continue;
}
Q_EMIT(tagDownloaded(tag));
}
}
void KonachanDanbooruService::processPostList(KJob *job)
{
qCDebug(LIBDANBOORU) << "Got post data OK";
if (job->error()) {
Q_EMIT(downloadError(job->errorString()));
}
StoredTransferJob *jobResult = qobject_cast<StoredTransferJob *>(job);
if (jobResult == 0) {
Q_EMIT(downloadError(QString("Internal error")));
return;
}
QByteArray data = jobResult->data();
bool ok;
bool needsXML = job->property("needsXML").toBool();
QList<QVariant> postList;
if (needsXML) {
// Special cases for pools
postList = parseDanbooruResult(data, QString("post"), &ok);
} else {
postList = parseDanbooruResult(data, &ok).toList();
}
if (!ok) {
Q_EMIT(downloadError(QString("Unable to decode data")));
return;
}
// How many posts do we have to fetch?
if (postList.isEmpty()) {
qCDebug(LIBDANBOORU) << "No posts found";
Q_EMIT(postDownloadFinished());
return;
}
m_postsToFetch = postList.length();
qCDebug(LIBDANBOORU) << "Found " << m_postsToFetch << "posts to fetch" << "with limit" << m_maxPosts;
// This is mostly needed for pools
if (postList.length() > m_maxPosts) {
m_postsToFetch = m_maxPosts;
postList = postList.mid(0, m_maxPosts);
}
for (auto element : postList) {
QVariantMap map = element.toMap();
DanbooruPost *post = new DanbooruPost(map);
// Remove unwanted posts
if (isPostBlacklisted(post, m_blacklist, m_maxRating)) {
m_postsToFetch--;
delete post;
continue;
}
QPixmap pix;
qCDebug(LIBDANBOORU) << "About to donwload images";
if (m_cache && m_cache->findPixmap(post->thumbnailUrl().url(), &pix)) {
post->setPixmap(pix);
Q_EMIT(postDownloaded(post));
m_postsToFetch--;
if (m_postsToFetch == 0) {
qCDebug(LIBDANBOORU) << "Post download finished";
Q_EMIT(postDownloadFinished());
}
} else {
// qCDebug(LIBDANBOORU) << "Downloading image" << post->thumbnailUrl();
StoredTransferJob *pixmapJob = KIO::storedGet(post->thumbnailUrl(),
KIO::NoReload, KIO::HideProgressInfo);
// We don't want to overload the servers, so set some rational
// priority
KIO::Scheduler::setJobPriority(static_cast<KIO::SimpleJob *>(job), 1);
QVariant variant;
variant.setValue(post);
connect(pixmapJob, &StoredTransferJob::result, [post, this, pix](KJob * job) mutable {
if (job->error())
{
Q_EMIT(downloadError(job->errorString()));
return;
}
StoredTransferJob *jobResult = qobject_cast<StoredTransferJob *>(job);
if (!pix.loadFromData(jobResult->data()))
{
Q_EMIT(downloadError(QString("Pixmap data could not be loaded")));
return;
}
post->setPixmap(pix);
if (m_cache)
{
//qCDebug(LIBDANBOORU) << "Inserting item in cache";
m_cache->insertPixmap(post->thumbnailUrl().url(), pix);
}
m_postsToFetch--; // One less post to do
qCDebug(LIBDANBOORU) << "Current posts remaining: " << m_postsToFetch;
Q_EMIT(postDownloaded(post));
if (m_postsToFetch == 0)
{
qCDebug(LIBDANBOORU) << "Post download finished";
Q_EMIT(postDownloadFinished());
}
});
}
}
}
void KonachanDanbooruService::downloadAllTags(KJob* job) {
Q_UNUSED(job);
}
} //namespace Danbooru