diff --git a/src/libdanbooru/danbooru_donmai.cpp b/src/libdanbooru/danbooru_donmai.cpp new file mode 100644 index 0000000..9a52325 --- /dev/null +++ b/src/libdanbooru/danbooru_donmai.cpp @@ -0,0 +1,620 @@ +/* + * Copyright 2015 Luca Beltrame + * + * This file is part of Danbooru Client. + * + * Danbooru Client is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Danbooru Client is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Danbooru Client. If not, see . + */ + +#include "danbooru_donmai.h" +#include "danboorupost.h" +#include "danboorupool.h" +#include "danboorutag.h" +#include "utils.h" +#include "libdanbooru_debug.h" + +// KF5 + +#include +#include +#include + +using KIO::StoredTransferJob; + +namespace Danbooru { + +/////// +// URIs +/////// + +const QLatin1String DanbooruService::postUri() const { + return QLatin1String("posts.json"); +} + +const QLatin1String DanbooruService::poolUri() const { + return QLatin1String("pools.json"); +} + +const QLatin1String DanbooruService::artistUri() const { + return QLatin1String("artists.json"); +} + +const QLatin1String DanbooruService::tagUri() const { + return QLatin1String("tags.json"); +} + +const QLatin1String DanbooruService::poolDataUri() const { + return QLatin1String("pools/%1.json"); +} + +const QLatin1String DanbooruService::relatedTagUri() const { + return QLatin1String("r.json"); +} + +//////////////// +// Other methods +//////////////// + + +Danbooru::ApiType DanbooruService::apiType() const { + return Danbooru::ApiType::Danbooru; +} + +////////////// +// API methods +////////////// + +void DanbooruService::getPostList() { + // We can't fetch more than 100 items, API limitation + + QMap parameters; + + parameters.insert("limit", QString::number(m_maxPosts)); + parameters.insert("page", QString::number(m_currentPage)); + + QUrl danbooruUrl = requestUrl(m_url, postUri(), m_username, + m_password, parameters, m_tags); + + qCDebug(LIBDANBOORU) << "Final constructed post URL" << danbooruUrl; + + KIO::StoredTransferJob *job = KIO::storedGet(danbooruUrl, KIO::NoReload, + KIO::HideProgressInfo); + + connect(job, &KIO::StoredTransferJob::result, this, &DanbooruService::processPostList); + +} + +void DanbooruService::getPoolList(int limit) +{ + + QUrl danbooruUrl; + + if (m_currentPage == 1) { + danbooruUrl = requestUrl(m_url, poolUri(), m_username, m_password); + } else { + QMap map; + map.insert("page", QString::number(m_currentPage)); + + danbooruUrl = requestUrl(m_url, poolUri(), m_username, + m_password, map); + } + + qCDebug(LIBDANBOORU) << "Final constructed pool list URL" << danbooruUrl.url(); + + KIO::StoredTransferJob *job = KIO::storedGet(danbooruUrl, KIO::NoReload, + KIO::HideProgressInfo); + + connect(job, &KIO::StoredTransferJob::result, [this, limit](KJob * job) { + + if (job->error()) { + Q_EMIT(downloadError(job->errorString())); + return; + } + + StoredTransferJob *jobResult = qobject_cast(job); + QByteArray data = jobResult->data(); + + bool ok; + QList poolList = parseDanbooruResult(data, &ok).toList(); + + if (!ok) { + Q_EMIT(downloadError(QString("Unable to decode data"))); + return; + } + + if (limit > 0 && poolList.length() > limit) { + poolList = poolList.mid(0, limit); + } + + for (auto element : qAsConst(poolList)) { + QVariantMap map = element.toMap(); + + DanbooruPool *pool = new DanbooruPool(map); + Q_EMIT(poolDownloaded(pool)); + } + + qCDebug(LIBDANBOORU) << "Pool download finished"; + Q_EMIT(poolDownloadFinished()); + } + + ); + +} + +void DanbooruService::getPool(int poolId, int page) +{ + + QMap parameters; + + if (page > 1) { + parameters.insert("page", QString::number(page)); + } + + const QString dataUri = QString(poolDataUri()).arg(poolId); + + QUrl danbooruUrl = requestUrl(m_url, dataUri, m_username, + m_password, parameters); + + qCDebug(LIBDANBOORU) << "Final constructed URL pool" << danbooruUrl; + + KIO::StoredTransferJob *job = KIO::storedGet(danbooruUrl, KIO::NoReload, + KIO::HideProgressInfo); + + connect(job, &StoredTransferJob::result, [this](KJob * job) { + + if (job->error()) { + Q_EMIT(downloadError(job->errorString())); + return; + } + + StoredTransferJob *jobResult = qobject_cast(job); + QByteArray data = jobResult->data(); + bool ok; + const QList postList = parseResult(data, apiType(), Pool, &ok); + + if (!ok) { + Q_EMIT(downloadError(QString("Unable to decode data"))); + return; + } + + const auto postIds = postList.at(0).value("raw_post_data").toList(); + + m_postsToFetch = postIds.length(); + + for (const auto postId: postIds) { + QUrl postUrl = requestUrl(m_url, QString("posts/%1.json").arg(postId.toString()), m_username, + m_password); + StoredTransferJob* postJob = KIO::storedGet(postUrl, KIO::NoReload, + KIO::HideProgressInfo); + + connect(postJob, &StoredTransferJob::result, this, &DanbooruService::processSinglePost); + } + }); +} + +void DanbooruService::getTagList(int limit, QString name) +{ + QMap parameters; + parameters.insert("limit", QString::number(limit)); + + if (!name.isEmpty()) { + parameters.insert("name", name); + } + parameters.insert("order", "date"); + + QUrl danbooruUrl = requestUrl(m_url, tagUri(), m_username, m_password, + parameters); +// qCDebug(LIBDANBOORU) << "Final constructed tag URL" << danbooruUrl.url(); + + KIO::StoredTransferJob *job = KIO::storedGet(danbooruUrl, KIO::NoReload, + KIO::HideProgressInfo); + connect(job, &KIO::StoredTransferJob::result, this, &DanbooruService::processTagList); + +} + +void DanbooruService::getRelatedTags(const QStringList &tags, + DanbooruTag::TagType tagType) +{ + + QString type; + switch (tagType) { + case DanbooruTag::General: + type = "general"; + break; + case DanbooruTag::Artist: + type = "artist"; + break; + case DanbooruTag::Copyright: + type = "copyright"; + break; + case DanbooruTag::Character: + type = "character"; + break; + case DanbooruTag::Unknown: + type = "unknown"; + break; + } + + QMap parameters; + parameters.insert("type", type); + + QUrl danbooruUrl = requestUrl(m_url, relatedTagUri(), m_username, + m_password, parameters, tags); + + // qCDebug(LIBDANBOORU) << "Final constructed related tag URL" << danbooruUrl; + + StoredTransferJob *job = KIO::storedGet( + danbooruUrl, KIO::NoReload, + KIO::HideProgressInfo + ); + + connect(job, &StoredTransferJob::result, [this](KJob * job) { + + if (job->error()) { + Q_EMIT(downloadError(job->errorString())); + return; + } + + StoredTransferJob *jobResult = qobject_cast(job); + QByteArray data = jobResult->data(); + bool ok; + + QVariantMap tagList = parseDanbooruResult(data, &ok).toMap(); + + if (!ok) { + Q_EMIT(downloadError(QString("Unable to decode data"))); + return; + } + + QVariantMap::const_iterator iter; + + // The service returns a list of key-related tag list pair, + // we iterate through them and remove the empty (not found) ones, then + // we call getTagList. Unfortunately Danbooru doesn't have a method to + // fetch all tags in batch, so this is done one by one. + + for (iter = tagList.constBegin(); iter != tagList.constEnd(); ++iter) { + + QList tags = iter.value().toList(); + + if (tags.isEmpty()) { + continue; + } + + for (auto tag : tags) { + // We get the first element in the list, the second is + // the ID which is useless (no API methods in Danbooru) + QString tagName = tag.toList().at(0).toString(); + getTagList(1, tagName); + } + + } + + } + + ); + +} + +//////// +// Slots +//////// + +void DanbooruService::processTagList(KJob *job) +{ + + if (job->error()) { + Q_EMIT(downloadError(job->errorString())); + return; + } + + StoredTransferJob *jobResult = qobject_cast(job); + QByteArray data = jobResult->data(); + + bool ok; + + // Most Danbooru implementations return tags in wrong order when + // using JSON, so we have to fall back to XML + const QList tagList = parseDanbooruResult(data, "tag", &ok); + + if (!ok) { + Q_EMIT(downloadError(QString("Unable to decode data"))); + return; + } + + for (auto element : tagList) { + QVariantMap map = element.toMap(); + DanbooruTag *tag = new DanbooruTag(map); + + if (!tag) { + continue; + } + + Q_EMIT(tagDownloaded(tag)); + } +} + +void DanbooruService::processPostList(KJob *job) +{ + + qCDebug(LIBDANBOORU) << "Got post data OK"; + + if (job->error()) { + Q_EMIT(downloadError(job->errorString())); + } + + StoredTransferJob *jobResult = qobject_cast(job); + + if (jobResult == 0) { + Q_EMIT(downloadError(QString("Internal error"))); + return; + + } + + QByteArray data = jobResult->data(); + + bool ok; + bool is_pool = job->property("is_pool").toBool(); + + QList postList; + + if (is_pool) { + // Special cases for pools + QVariantMap postMap = parseResult(data, apiType(), Danbooru::Pool, &ok).at(0); + auto postData = postMap.value("raw_post_data").toList(); + for (const auto post: postData) { + postList.append(extractPostData(post, apiType())); + } + + } else { + postList = parseResult(data, apiType(), Danbooru::Post, &ok); + } + + if (!ok) { + Q_EMIT(downloadError(QString("Unable to decode data"))); + return; + } + + // How many posts do we have to fetch? + + if (postList.isEmpty()) { + qCDebug(LIBDANBOORU) << "No posts found"; + Q_EMIT(postDownloadFinished()); + return; + } + + m_postsToFetch = postList.length(); + qCDebug(LIBDANBOORU) << "Found " << m_postsToFetch << "posts to fetch" << "with limit" << m_maxPosts; + + // This is mostly needed for pools + if (postList.length() > m_maxPosts) { + m_postsToFetch = m_maxPosts; + postList = postList.mid(0, m_maxPosts); + } + + for (const auto element : qAsConst(postList)) { + + DanbooruPost *post = new DanbooruPost(element); + + // Remove unwanted posts + + if (isPostBlacklisted(post, m_blacklist, m_maxRating)) { + m_postsToFetch--; + delete post; + continue; + } + + QPixmap pix; + + qCDebug(LIBDANBOORU) << "About to download images"; + + if (m_cache && m_cache->findPixmap(post->thumbnailUrl().url(), &pix)) { + + qDebug() << "reimplement"; + post->setPixmap(pix); + Q_EMIT(postDownloaded(post)); + m_postsToFetch--; + + if (m_postsToFetch == 0) { + qCDebug(LIBDANBOORU) << "Post download finished"; + Q_EMIT(postDownloadFinished()); + } + + } else { + + qCDebug(LIBDANBOORU) << "Downloading image" << post->thumbnailUrl(); + StoredTransferJob *pixmapJob = KIO::storedGet(post->thumbnailUrl(), + KIO::NoReload, KIO::HideProgressInfo); + + // We don't want to overload the servers, so set some rational + // priority + + KIO::Scheduler::setJobPriority(static_cast(pixmapJob), 1); + + QVariant variant; + variant.setValue(post); + + connect(pixmapJob, &StoredTransferJob::result, this, [post, this, pix](KJob * job) mutable { + + if (job->error()) + { + Q_EMIT(downloadError(job->errorString())); + return; + } + + StoredTransferJob *jobResult = qobject_cast(job); + + if (!pix.loadFromData(jobResult->data())) + { + Q_EMIT(downloadError(QString("Pixmap data could not be loaded"))); + return; + } + + post->setPixmap(pix); + + if (m_cache) + { + //qCDebug(LIBDANBOORU) << "Inserting item in cache"; + m_cache->insertPixmap(post->thumbnailUrl().url(), pix); + } + + m_postsToFetch--; // One less post to do + + qCDebug(LIBDANBOORU) << "Current posts remaining: " << m_postsToFetch; + Q_EMIT(postDownloaded(post)); + + if (m_postsToFetch == 0) + { + qCDebug(LIBDANBOORU) << "Post download finished"; + Q_EMIT(postDownloadFinished()); + } + + }); + + } + + } + +} + +void DanbooruService::processSinglePost(KJob* job) { + + if (job->error()) + { + Q_EMIT(downloadError(job->errorString())); + return; + } + + StoredTransferJob *jobResult = qobject_cast(job); + + if (jobResult == 0) { + Q_EMIT(downloadError(QString("Internal error"))); + return; + + } + + QByteArray data = jobResult->data(); + bool ok; + + QList postList; + postList = parseResult(data, apiType(), Danbooru::Post, &ok); + if (!ok) { + Q_EMIT(downloadError(QString("Unable to decode data"))); + return; + } + + // How many posts do we have to fetch? + + if (postList.isEmpty()) { + qCDebug(LIBDANBOORU) << "No post found"; + Q_EMIT(postDownloadFinished()); + return; + } + + DanbooruPost* post = new DanbooruPost(postList.at(0)); + if (isPostBlacklisted(post, m_blacklist, m_maxRating)) { + m_postsToFetch--; + delete post; + return; + } + + QPixmap pix; + qCDebug(LIBDANBOORU) << "About to download image"; + + if (m_cache && m_cache->findPixmap(post->thumbnailUrl().url(), &pix)) { + + post->setPixmap(pix); + Q_EMIT(postDownloaded(post)); + m_postsToFetch--; + + if (m_postsToFetch == 0) { + qCDebug(LIBDANBOORU) << "Post download finished"; + Q_EMIT(postDownloadFinished()); + } + + } else { + StoredTransferJob *pixmapJob = KIO::storedGet(post->thumbnailUrl(), + KIO::NoReload, KIO::HideProgressInfo); + + // We don't want to overload the servers, so set some rational + // priority + + KIO::Scheduler::setJobPriority(static_cast(pixmapJob), 1); + + QVariant variant; + variant.setValue(post); + + pixmapJob->setProperty("post", variant); + pixmapJob->setProperty("pixmap", pix); + connect(pixmapJob, &StoredTransferJob::result, this, &DanbooruService::processPixmap); + } + + +} + +void DanbooruService::processPixmap(KJob* job) { + + if (job->error()) + { + Q_EMIT(downloadError(job->errorString())); + return; + } + + StoredTransferJob *jobResult = qobject_cast(job); + + if (jobResult == 0) { + Q_EMIT(downloadError(QString("Internal error"))); + return; + + } + + QByteArray data = jobResult->data(); + auto post = job->property("post").value(); + auto pix = job->property("pixmap").value(); + + if (!pix.loadFromData(jobResult->data())) + { + Q_EMIT(downloadError(QString("Pixmap data could not be loaded"))); + return; + } + + post->setPixmap(pix); + + if (m_cache) + { + //qCDebug(LIBDANBOORU) << "Inserting item in cache"; + m_cache->insertPixmap(post->thumbnailUrl().url(), pix); + } + + m_postsToFetch--; // One less post to do + + qCDebug(LIBDANBOORU) << "Current posts remaining: " << m_postsToFetch; + Q_EMIT(postDownloaded(post)); + + if (m_postsToFetch == 0) + { + qCDebug(LIBDANBOORU) << "Post download finished"; + Q_EMIT(postDownloadFinished()); + } + +} + +void DanbooruService::downloadAllTags(KJob* job) { + Q_UNUSED(job); +} + +} //namespace Danbooru + + + diff --git a/src/libdanbooru/danbooru_donmai.h b/src/libdanbooru/danbooru_donmai.h new file mode 100644 index 0000000..7bd01f8 --- /dev/null +++ b/src/libdanbooru/danbooru_donmai.h @@ -0,0 +1,68 @@ +/* + * Copyright 2015 Luca Beltrame + * + * This file is part of Danbooru Client. + * + * Danbooru Client is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Danbooru Client is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Danbooru Client. If not, see . + */ + +#ifndef DANBOORU_DONMAI_H +#define DANBOORU_DONMAI_H + +#include "servicebase.h" +#include "danbooru.h" + +class KJob; + +namespace Danbooru { + +class DanbooruService: public DanbooruServiceBase { + +Q_OBJECT + +private: + const QLatin1String postUri() const override; + const QLatin1String poolUri() const override; + const QLatin1String artistUri() const override; + const QLatin1String tagUri() const override; + const QLatin1String poolDataUri() const override; + const QLatin1String relatedTagUri() const override; + +public: + + using DanbooruServiceBase::DanbooruServiceBase; // superclass constructor + + Danbooru::ApiType apiType() const override; + + void getPostList() override; + void getPoolList(int limit = -1) override; + void getPool(int poolId, int page = 1) override; + void getTagList(int limit = 10, QString name = "") override; + void getRelatedTags(const QStringList &tags, + DanbooruTag::TagType tagType = DanbooruTag::General) override; + +private Q_SLOTS: + void processPostList(KJob *job); + void processTagList(KJob *job); + void downloadAllTags(KJob *job); + void processSinglePost(KJob *job); + void processPixmap(KJob *job); + + +}; + +} // namespace Danbooru + +#endif +