Revert "[WIP] Fallback on 404" (#3364)
This commit is contained in:
parent
e09ea1e683
commit
e341337ce0
2 changed files with 45 additions and 126 deletions
|
@ -43,96 +43,22 @@ public:
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
PictureToLoad::PictureToLoad(CardInfoPtr _card) :
|
PictureToLoad::PictureToLoad(CardInfoPtr _card) : card(std::move(_card)), setIndex(0)
|
||||||
card(std::move(_card)),
|
|
||||||
setIndex(0),
|
|
||||||
urlIndex(0)
|
|
||||||
{
|
{
|
||||||
// This will be replaced with an expandable list ideally
|
|
||||||
urlTemplates.append(settingsCache->getPicUrl());
|
|
||||||
urlTemplates.append(settingsCache->getPicUrlFallback());
|
|
||||||
|
|
||||||
if (card) {
|
if (card) {
|
||||||
sortedSets = card->getSets();
|
sortedSets = card->getSets();
|
||||||
qSort(sortedSets.begin(), sortedSets.end(), SetDownloadPriorityComparator());
|
qSort(sortedSets.begin(), sortedSets.end(), SetDownloadPriorityComparator());
|
||||||
populateSetUrls();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void PictureToLoad::populateSetUrls()
|
|
||||||
{
|
|
||||||
currentSetUrls.clear();
|
|
||||||
|
|
||||||
QString setCustomURL = card->getCustomPicURL(getCurrentSet()->getShortName());
|
|
||||||
|
|
||||||
if (!setCustomURL.isEmpty()) {
|
|
||||||
currentSetUrls.append(setCustomURL);
|
|
||||||
}
|
|
||||||
|
|
||||||
for (int i; i< urlTemplates.size(); i++) {
|
|
||||||
QString transformedUrl = transformUrl(urlTemplates[i]);
|
|
||||||
|
|
||||||
if (!transformedUrl.isEmpty()) {
|
|
||||||
currentSetUrls.append(transformedUrl);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
urlIndex = 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
bool PictureToLoad::nextSet()
|
bool PictureToLoad::nextSet()
|
||||||
{
|
{
|
||||||
if (setIndex == sortedSets.size() - 1)
|
if (setIndex == sortedSets.size() - 1)
|
||||||
return false;
|
return false;
|
||||||
++setIndex;
|
++setIndex;
|
||||||
populateSetUrls();
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool PictureToLoad::nextUrl()
|
|
||||||
{
|
|
||||||
/* If we are past the list of urls currently populated
|
|
||||||
* for this set, try to move to the next set. This
|
|
||||||
* will repopulate the list of urls.
|
|
||||||
*/
|
|
||||||
if (urlIndex == currentSetUrls.size() - 1) {
|
|
||||||
if (nextSet()){
|
|
||||||
// There is a new set to check, so test the Url again
|
|
||||||
// UrlIndex would have been reset inside nextSet()
|
|
||||||
if (urlIndex == currentSetUrls.size() - 1) {
|
|
||||||
// The newly populated set did not yield any usable Urls.
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
// Set was updated, UrlIndex is reset, proceed checking Urls.
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
// Because there is not another set, there are not more Urls to check.
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
// We are still in the middle of the list, increment and return
|
|
||||||
// This must be inside the else to protect against UrlIndex being incremented
|
|
||||||
// when the set is moved to the next set.
|
|
||||||
++urlIndex;
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
QString PictureToLoad::getCurrentUrl() const
|
|
||||||
{
|
|
||||||
if (urlIndex < currentSetUrls.size())
|
|
||||||
return currentSetUrls[urlIndex];
|
|
||||||
else
|
|
||||||
return QString("");
|
|
||||||
}
|
|
||||||
|
|
||||||
QString PictureToLoad::getSetName() const
|
QString PictureToLoad::getSetName() const
|
||||||
{
|
{
|
||||||
if (setIndex < sortedSets.size())
|
if (setIndex < sortedSets.size())
|
||||||
|
@ -195,13 +121,13 @@ void PictureLoaderWorker::processLoadQueue()
|
||||||
QString setName = cardBeingLoaded.getSetName();
|
QString setName = cardBeingLoaded.getSetName();
|
||||||
QString cardName = cardBeingLoaded.getCard()->getName();
|
QString cardName = cardBeingLoaded.getCard()->getName();
|
||||||
QString correctedCardName = cardBeingLoaded.getCard()->getCorrectedName();
|
QString correctedCardName = cardBeingLoaded.getCard()->getCorrectedName();
|
||||||
qDebug() << "Trying to load picture for card:" << cardName << " from set:" << setName;
|
qDebug() << "Trying to load picture (set: " << setName << " card: " << cardName << ")";
|
||||||
|
|
||||||
if (cardImageExistsOnDisk(setName, correctedCardName))
|
if (cardImageExistsOnDisk(setName, correctedCardName))
|
||||||
continue;
|
continue;
|
||||||
|
|
||||||
if (picDownload) {
|
if (picDownload) {
|
||||||
qDebug() << "Picture not found, trying to download";
|
qDebug() << "Picture NOT found, trying to download (set: " << setName << " card: " << cardName << ")";
|
||||||
cardsToDownload.append(cardBeingLoaded);
|
cardsToDownload.append(cardBeingLoaded);
|
||||||
cardBeingLoaded.clear();
|
cardBeingLoaded.clear();
|
||||||
if (!downloadRunning)
|
if (!downloadRunning)
|
||||||
|
@ -271,46 +197,51 @@ bool PictureLoaderWorker::cardImageExistsOnDisk(QString &setName, QString &corre
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
QString PictureToLoad::transformUrl(QString urlTemplate) const
|
QString PictureLoaderWorker::getPicUrl()
|
||||||
{
|
{
|
||||||
QString transformedUrl = urlTemplate;
|
if (!picDownload)
|
||||||
CardSetPtr set = getCurrentSet();
|
return QString();
|
||||||
|
|
||||||
|
CardInfoPtr card = cardBeingDownloaded.getCard();
|
||||||
|
CardSetPtr set = cardBeingDownloaded.getCurrentSet();
|
||||||
|
QString picUrl = QString("");
|
||||||
|
|
||||||
|
// if sets have been defined for the card, they can contain custom picUrls
|
||||||
|
if (set) {
|
||||||
|
picUrl = card->getCustomPicURL(set->getShortName());
|
||||||
|
if (!picUrl.isEmpty())
|
||||||
|
return picUrl;
|
||||||
|
}
|
||||||
|
|
||||||
// if a card has a muid, use the default url; if not, use the fallback
|
// if a card has a muid, use the default url; if not, use the fallback
|
||||||
int muid = set ? card->getMuId(set->getShortName()) : 0;
|
int muid = set ? card->getMuId(set->getShortName()) : 0;
|
||||||
|
picUrl = muid ? settingsCache->getPicUrl() : settingsCache->getPicUrlFallback();
|
||||||
|
|
||||||
transformedUrl.replace("!name!", QUrl::toPercentEncoding(card->getName()));
|
picUrl.replace("!name!", QUrl::toPercentEncoding(card->getName()));
|
||||||
transformedUrl.replace("!name_lower!", QUrl::toPercentEncoding(card->getName().toLower()));
|
picUrl.replace("!name_lower!", QUrl::toPercentEncoding(card->getName().toLower()));
|
||||||
transformedUrl.replace("!corrected_name!", QUrl::toPercentEncoding(card->getCorrectedName()));
|
picUrl.replace("!corrected_name!", QUrl::toPercentEncoding(card->getCorrectedName()));
|
||||||
transformedUrl.replace("!corrected_name_lower!", QUrl::toPercentEncoding(card->getCorrectedName().toLower()));
|
picUrl.replace("!corrected_name_lower!", QUrl::toPercentEncoding(card->getCorrectedName().toLower()));
|
||||||
transformedUrl.replace("!cardid!", QUrl::toPercentEncoding(QString::number(muid)));
|
picUrl.replace("!cardid!", QUrl::toPercentEncoding(QString::number(muid)));
|
||||||
|
|
||||||
if (set) {
|
if (set) {
|
||||||
// renamed from !setnumber! to !collectornumber! on 20160819. Remove the old one when convenient.
|
// renamed from !setnumber! to !collectornumber! on 20160819. Remove the old one when convenient.
|
||||||
transformedUrl.replace("!setnumber!", QUrl::toPercentEncoding(card->getCollectorNumber(set->getShortName())));
|
picUrl.replace("!setnumber!", QUrl::toPercentEncoding(card->getCollectorNumber(set->getShortName())));
|
||||||
transformedUrl.replace("!collectornumber!", QUrl::toPercentEncoding(card->getCollectorNumber(set->getShortName())));
|
picUrl.replace("!collectornumber!", QUrl::toPercentEncoding(card->getCollectorNumber(set->getShortName())));
|
||||||
|
|
||||||
transformedUrl.replace("!setcode!", QUrl::toPercentEncoding(set->getShortName()));
|
picUrl.replace("!setcode!", QUrl::toPercentEncoding(set->getShortName()));
|
||||||
transformedUrl.replace("!setcode_lower!", QUrl::toPercentEncoding(set->getShortName().toLower()));
|
picUrl.replace("!setcode_lower!", QUrl::toPercentEncoding(set->getShortName().toLower()));
|
||||||
transformedUrl.replace("!setname!", QUrl::toPercentEncoding(set->getLongName()));
|
picUrl.replace("!setname!", QUrl::toPercentEncoding(set->getLongName()));
|
||||||
transformedUrl.replace("!setname_lower!", QUrl::toPercentEncoding(set->getLongName().toLower()));
|
picUrl.replace("!setname_lower!", QUrl::toPercentEncoding(set->getLongName().toLower()));
|
||||||
}
|
}
|
||||||
|
|
||||||
if (transformedUrl.contains("!name!") ||
|
if (picUrl.contains("!name!") || picUrl.contains("!name_lower!") || picUrl.contains("!corrected_name!") ||
|
||||||
transformedUrl.contains("!name_lower!") ||
|
picUrl.contains("!corrected_name_lower!") || picUrl.contains("!setnumber!") || picUrl.contains("!setcode!") ||
|
||||||
transformedUrl.contains("!corrected_name!") ||
|
picUrl.contains("!setcode_lower!") || picUrl.contains("!setname!") || picUrl.contains("!setname_lower!") ||
|
||||||
transformedUrl.contains("!corrected_name_lower!") ||
|
picUrl.contains("!cardid!")) {
|
||||||
transformedUrl.contains("!setnumber!") ||
|
qDebug() << "Insufficient card data to download" << card->getName() << "Url:" << picUrl;
|
||||||
transformedUrl.contains("!setcode!") ||
|
|
||||||
transformedUrl.contains("!setcode_lower!") ||
|
|
||||||
transformedUrl.contains("!setname!") ||
|
|
||||||
transformedUrl.contains("!setname_lower!") ||
|
|
||||||
transformedUrl.contains("!cardid!")) {
|
|
||||||
qDebug() << "Insufficient card data to download" << card->getName() << "Url:" << urlTemplates[urlIndex];
|
|
||||||
return QString();
|
return QString();
|
||||||
}
|
}
|
||||||
|
|
||||||
return transformedUrl;
|
return picUrl;
|
||||||
}
|
}
|
||||||
|
|
||||||
void PictureLoaderWorker::startNextPicDownload()
|
void PictureLoaderWorker::startNextPicDownload()
|
||||||
|
@ -325,33 +256,30 @@ void PictureLoaderWorker::startNextPicDownload()
|
||||||
|
|
||||||
cardBeingDownloaded = cardsToDownload.takeFirst();
|
cardBeingDownloaded = cardsToDownload.takeFirst();
|
||||||
|
|
||||||
QString picUrl = cardBeingDownloaded.getCurrentUrl();
|
QString picUrl = getPicUrl();
|
||||||
|
|
||||||
if (picUrl.isEmpty()) {
|
if (picUrl.isEmpty()) {
|
||||||
downloadRunning = false;
|
downloadRunning = false;
|
||||||
picDownloadFailed();
|
picDownloadFailed();
|
||||||
} else {
|
} else {
|
||||||
QUrl url(picUrl); // For now, just use the first one, like always.
|
QUrl url(picUrl);
|
||||||
|
|
||||||
QNetworkRequest req(url);
|
QNetworkRequest req(url);
|
||||||
qDebug() << "Trying to download picture for card:" << cardBeingDownloaded.getCard()->getName()
|
qDebug() << "starting picture download:" << cardBeingDownloaded.getCard()->getName() << "Url:" << req.url();
|
||||||
<< " from set:" << cardBeingDownloaded.getSetName() << "from url:" << picUrl;
|
|
||||||
networkManager->get(req);
|
networkManager->get(req);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void PictureLoaderWorker::picDownloadFailed()
|
void PictureLoaderWorker::picDownloadFailed()
|
||||||
{
|
{
|
||||||
if (cardBeingDownloaded.nextUrl()) {
|
if (cardBeingDownloaded.nextSet()) {
|
||||||
//qDebug() << "Picture NOT found, download failed, moving to next url (url: " << cardBeingDownloaded.getCurrentUrl()
|
qDebug() << "Picture NOT found, download failed, moving to next set (newset: "
|
||||||
// << " set: " << cardBeingDownloaded.getSetName()
|
<< cardBeingDownloaded.getSetName() << " card: " << cardBeingDownloaded.getCard()->getName() << ")";
|
||||||
// << " card: " << cardBeingDownloaded.getCard()->getName() << ")";
|
|
||||||
mutex.lock();
|
mutex.lock();
|
||||||
loadQueue.prepend(cardBeingDownloaded);
|
loadQueue.prepend(cardBeingDownloaded);
|
||||||
mutex.unlock();
|
mutex.unlock();
|
||||||
} else {
|
} else {
|
||||||
qDebug() << "Picture NOT found, download failed, no more url combinations to try: BAILING OUT for card: "
|
qDebug() << "Picture NOT found, download failed, no more sets to try: BAILING OUT (oldset: "
|
||||||
<< cardBeingDownloaded.getCard()->getName() << ")";
|
<< cardBeingDownloaded.getSetName() << " card: " << cardBeingDownloaded.getCard()->getName() << ")";
|
||||||
imageLoaded(cardBeingDownloaded.getCard(), QImage());
|
imageLoaded(cardBeingDownloaded.getCard(), QImage());
|
||||||
cardBeingDownloaded.clear();
|
cardBeingDownloaded.clear();
|
||||||
}
|
}
|
||||||
|
@ -374,7 +302,7 @@ void PictureLoaderWorker::picDownloadFinished(QNetworkReply *reply)
|
||||||
if (statusCode == 301 || statusCode == 302) {
|
if (statusCode == 301 || statusCode == 302) {
|
||||||
QUrl redirectUrl = reply->attribute(QNetworkRequest::RedirectionTargetAttribute).toUrl();
|
QUrl redirectUrl = reply->attribute(QNetworkRequest::RedirectionTargetAttribute).toUrl();
|
||||||
QNetworkRequest req(redirectUrl);
|
QNetworkRequest req(redirectUrl);
|
||||||
qDebug() << "following redirect:" << cardBeingDownloaded.getCard()->getName() << "Url:" << req.url().toString();
|
qDebug() << "following redirect:" << cardBeingDownloaded.getCard()->getName() << "Url:" << req.url();
|
||||||
networkManager->get(req);
|
networkManager->get(req);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -417,9 +345,7 @@ void PictureLoaderWorker::picDownloadFinished(QNetworkReply *reply)
|
||||||
}
|
}
|
||||||
|
|
||||||
imageLoaded(cardBeingDownloaded.getCard(), testImage);
|
imageLoaded(cardBeingDownloaded.getCard(), testImage);
|
||||||
qDebug() << "Image successfully downloaded from " << reply->request().url().toString();
|
|
||||||
} else {
|
} else {
|
||||||
qDebug() << "Possible picture at " << reply->request().url().toString() << " could not be loaded";
|
|
||||||
picDownloadFailed();
|
picDownloadFailed();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -18,10 +18,7 @@ private:
|
||||||
|
|
||||||
CardInfoPtr card;
|
CardInfoPtr card;
|
||||||
QList<CardSetPtr> sortedSets;
|
QList<CardSetPtr> sortedSets;
|
||||||
QList<QString> urlTemplates;
|
|
||||||
QList<QString> currentSetUrls;
|
|
||||||
int setIndex;
|
int setIndex;
|
||||||
int urlIndex;
|
|
||||||
|
|
||||||
public:
|
public:
|
||||||
PictureToLoad(CardInfoPtr _card = CardInfoPtr());
|
PictureToLoad(CardInfoPtr _card = CardInfoPtr());
|
||||||
|
@ -34,12 +31,8 @@ public:
|
||||||
card.clear();
|
card.clear();
|
||||||
}
|
}
|
||||||
CardSetPtr getCurrentSet() const;
|
CardSetPtr getCurrentSet() const;
|
||||||
QString getCurrentUrl() const;
|
|
||||||
QString getSetName() const;
|
QString getSetName() const;
|
||||||
QString transformUrl(QString urlTemplate) const;
|
|
||||||
bool nextSet();
|
bool nextSet();
|
||||||
bool nextUrl();
|
|
||||||
void populateSetUrls();
|
|
||||||
};
|
};
|
||||||
|
|
||||||
class PictureLoaderWorker : public QObject
|
class PictureLoaderWorker : public QObject
|
||||||
|
@ -64,7 +57,7 @@ private:
|
||||||
PictureToLoad cardBeingDownloaded;
|
PictureToLoad cardBeingDownloaded;
|
||||||
bool picDownload, downloadRunning, loadQueueRunning;
|
bool picDownload, downloadRunning, loadQueueRunning;
|
||||||
void startNextPicDownload();
|
void startNextPicDownload();
|
||||||
QList<QString> getAllPicUrls();
|
QString getPicUrl();
|
||||||
bool cardImageExistsOnDisk(QString &setName, QString &correctedCardname);
|
bool cardImageExistsOnDisk(QString &setName, QString &correctedCardname);
|
||||||
bool imageIsBlackListed(const QByteArray &picData);
|
bool imageIsBlackListed(const QByteArray &picData);
|
||||||
private slots:
|
private slots:
|
||||||
|
|
Loading…
Reference in a new issue