Revert "[WIP] Fallback on 404" (#3364)

This commit is contained in:
Zach H 2018-08-11 09:06:18 -04:00 committed by GitHub
parent e09ea1e683
commit e341337ce0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 45 additions and 126 deletions

View file

@ -43,96 +43,22 @@ public:
} }
}; };
PictureToLoad::PictureToLoad(CardInfoPtr _card) : PictureToLoad::PictureToLoad(CardInfoPtr _card) : card(std::move(_card)), setIndex(0)
card(std::move(_card)),
setIndex(0),
urlIndex(0)
{ {
// This will be replaced with an expandable list ideally
urlTemplates.append(settingsCache->getPicUrl());
urlTemplates.append(settingsCache->getPicUrlFallback());
if (card) { if (card) {
sortedSets = card->getSets(); sortedSets = card->getSets();
qSort(sortedSets.begin(), sortedSets.end(), SetDownloadPriorityComparator()); qSort(sortedSets.begin(), sortedSets.end(), SetDownloadPriorityComparator());
populateSetUrls();
} }
} }
void PictureToLoad::populateSetUrls()
{
currentSetUrls.clear();
QString setCustomURL = card->getCustomPicURL(getCurrentSet()->getShortName());
if (!setCustomURL.isEmpty()) {
currentSetUrls.append(setCustomURL);
}
for (int i; i< urlTemplates.size(); i++) {
QString transformedUrl = transformUrl(urlTemplates[i]);
if (!transformedUrl.isEmpty()) {
currentSetUrls.append(transformedUrl);
}
}
urlIndex = 0;
}
bool PictureToLoad::nextSet() bool PictureToLoad::nextSet()
{ {
if (setIndex == sortedSets.size() - 1) if (setIndex == sortedSets.size() - 1)
return false; return false;
++setIndex; ++setIndex;
populateSetUrls();
return true; return true;
} }
bool PictureToLoad::nextUrl()
{
/* If we are past the list of urls currently populated
* for this set, try to move to the next set. This
* will repopulate the list of urls.
*/
if (urlIndex == currentSetUrls.size() - 1) {
if (nextSet()){
// There is a new set to check, so test the Url again
// UrlIndex would have been reset inside nextSet()
if (urlIndex == currentSetUrls.size() - 1) {
// The newly populated set did not yield any usable Urls.
return false;
}
else
{
// Set was updated, UrlIndex is reset, proceed checking Urls.
return true;
}
}
else
{
// Because there is not another set, there are not more Urls to check.
return false;
}
}
else
{
// We are still in the middle of the list, increment and return
// This must be inside the else to protect against UrlIndex being incremented
// when the set is moved to the next set.
++urlIndex;
return true;
}
}
QString PictureToLoad::getCurrentUrl() const
{
if (urlIndex < currentSetUrls.size())
return currentSetUrls[urlIndex];
else
return QString("");
}
QString PictureToLoad::getSetName() const QString PictureToLoad::getSetName() const
{ {
if (setIndex < sortedSets.size()) if (setIndex < sortedSets.size())
@ -195,13 +121,13 @@ void PictureLoaderWorker::processLoadQueue()
QString setName = cardBeingLoaded.getSetName(); QString setName = cardBeingLoaded.getSetName();
QString cardName = cardBeingLoaded.getCard()->getName(); QString cardName = cardBeingLoaded.getCard()->getName();
QString correctedCardName = cardBeingLoaded.getCard()->getCorrectedName(); QString correctedCardName = cardBeingLoaded.getCard()->getCorrectedName();
qDebug() << "Trying to load picture for card:" << cardName << " from set:" << setName; qDebug() << "Trying to load picture (set: " << setName << " card: " << cardName << ")";
if (cardImageExistsOnDisk(setName, correctedCardName)) if (cardImageExistsOnDisk(setName, correctedCardName))
continue; continue;
if (picDownload) { if (picDownload) {
qDebug() << "Picture not found, trying to download"; qDebug() << "Picture NOT found, trying to download (set: " << setName << " card: " << cardName << ")";
cardsToDownload.append(cardBeingLoaded); cardsToDownload.append(cardBeingLoaded);
cardBeingLoaded.clear(); cardBeingLoaded.clear();
if (!downloadRunning) if (!downloadRunning)
@ -271,46 +197,51 @@ bool PictureLoaderWorker::cardImageExistsOnDisk(QString &setName, QString &corre
return false; return false;
} }
QString PictureToLoad::transformUrl(QString urlTemplate) const QString PictureLoaderWorker::getPicUrl()
{ {
QString transformedUrl = urlTemplate; if (!picDownload)
CardSetPtr set = getCurrentSet(); return QString();
CardInfoPtr card = cardBeingDownloaded.getCard();
CardSetPtr set = cardBeingDownloaded.getCurrentSet();
QString picUrl = QString("");
// if sets have been defined for the card, they can contain custom picUrls
if (set) {
picUrl = card->getCustomPicURL(set->getShortName());
if (!picUrl.isEmpty())
return picUrl;
}
// if a card has a muid, use the default url; if not, use the fallback // if a card has a muid, use the default url; if not, use the fallback
int muid = set ? card->getMuId(set->getShortName()) : 0; int muid = set ? card->getMuId(set->getShortName()) : 0;
picUrl = muid ? settingsCache->getPicUrl() : settingsCache->getPicUrlFallback();
transformedUrl.replace("!name!", QUrl::toPercentEncoding(card->getName())); picUrl.replace("!name!", QUrl::toPercentEncoding(card->getName()));
transformedUrl.replace("!name_lower!", QUrl::toPercentEncoding(card->getName().toLower())); picUrl.replace("!name_lower!", QUrl::toPercentEncoding(card->getName().toLower()));
transformedUrl.replace("!corrected_name!", QUrl::toPercentEncoding(card->getCorrectedName())); picUrl.replace("!corrected_name!", QUrl::toPercentEncoding(card->getCorrectedName()));
transformedUrl.replace("!corrected_name_lower!", QUrl::toPercentEncoding(card->getCorrectedName().toLower())); picUrl.replace("!corrected_name_lower!", QUrl::toPercentEncoding(card->getCorrectedName().toLower()));
transformedUrl.replace("!cardid!", QUrl::toPercentEncoding(QString::number(muid))); picUrl.replace("!cardid!", QUrl::toPercentEncoding(QString::number(muid)));
if (set) { if (set) {
// renamed from !setnumber! to !collectornumber! on 20160819. Remove the old one when convenient. // renamed from !setnumber! to !collectornumber! on 20160819. Remove the old one when convenient.
transformedUrl.replace("!setnumber!", QUrl::toPercentEncoding(card->getCollectorNumber(set->getShortName()))); picUrl.replace("!setnumber!", QUrl::toPercentEncoding(card->getCollectorNumber(set->getShortName())));
transformedUrl.replace("!collectornumber!", QUrl::toPercentEncoding(card->getCollectorNumber(set->getShortName()))); picUrl.replace("!collectornumber!", QUrl::toPercentEncoding(card->getCollectorNumber(set->getShortName())));
transformedUrl.replace("!setcode!", QUrl::toPercentEncoding(set->getShortName())); picUrl.replace("!setcode!", QUrl::toPercentEncoding(set->getShortName()));
transformedUrl.replace("!setcode_lower!", QUrl::toPercentEncoding(set->getShortName().toLower())); picUrl.replace("!setcode_lower!", QUrl::toPercentEncoding(set->getShortName().toLower()));
transformedUrl.replace("!setname!", QUrl::toPercentEncoding(set->getLongName())); picUrl.replace("!setname!", QUrl::toPercentEncoding(set->getLongName()));
transformedUrl.replace("!setname_lower!", QUrl::toPercentEncoding(set->getLongName().toLower())); picUrl.replace("!setname_lower!", QUrl::toPercentEncoding(set->getLongName().toLower()));
} }
if (transformedUrl.contains("!name!") || if (picUrl.contains("!name!") || picUrl.contains("!name_lower!") || picUrl.contains("!corrected_name!") ||
transformedUrl.contains("!name_lower!") || picUrl.contains("!corrected_name_lower!") || picUrl.contains("!setnumber!") || picUrl.contains("!setcode!") ||
transformedUrl.contains("!corrected_name!") || picUrl.contains("!setcode_lower!") || picUrl.contains("!setname!") || picUrl.contains("!setname_lower!") ||
transformedUrl.contains("!corrected_name_lower!") || picUrl.contains("!cardid!")) {
transformedUrl.contains("!setnumber!") || qDebug() << "Insufficient card data to download" << card->getName() << "Url:" << picUrl;
transformedUrl.contains("!setcode!") ||
transformedUrl.contains("!setcode_lower!") ||
transformedUrl.contains("!setname!") ||
transformedUrl.contains("!setname_lower!") ||
transformedUrl.contains("!cardid!")) {
qDebug() << "Insufficient card data to download" << card->getName() << "Url:" << urlTemplates[urlIndex];
return QString(); return QString();
} }
return transformedUrl; return picUrl;
} }
void PictureLoaderWorker::startNextPicDownload() void PictureLoaderWorker::startNextPicDownload()
@ -325,33 +256,30 @@ void PictureLoaderWorker::startNextPicDownload()
cardBeingDownloaded = cardsToDownload.takeFirst(); cardBeingDownloaded = cardsToDownload.takeFirst();
QString picUrl = cardBeingDownloaded.getCurrentUrl(); QString picUrl = getPicUrl();
if (picUrl.isEmpty()) { if (picUrl.isEmpty()) {
downloadRunning = false; downloadRunning = false;
picDownloadFailed(); picDownloadFailed();
} else { } else {
QUrl url(picUrl); // For now, just use the first one, like always. QUrl url(picUrl);
QNetworkRequest req(url); QNetworkRequest req(url);
qDebug() << "Trying to download picture for card:" << cardBeingDownloaded.getCard()->getName() qDebug() << "starting picture download:" << cardBeingDownloaded.getCard()->getName() << "Url:" << req.url();
<< " from set:" << cardBeingDownloaded.getSetName() << "from url:" << picUrl;
networkManager->get(req); networkManager->get(req);
} }
} }
void PictureLoaderWorker::picDownloadFailed() void PictureLoaderWorker::picDownloadFailed()
{ {
if (cardBeingDownloaded.nextUrl()) { if (cardBeingDownloaded.nextSet()) {
//qDebug() << "Picture NOT found, download failed, moving to next url (url: " << cardBeingDownloaded.getCurrentUrl() qDebug() << "Picture NOT found, download failed, moving to next set (newset: "
// << " set: " << cardBeingDownloaded.getSetName() << cardBeingDownloaded.getSetName() << " card: " << cardBeingDownloaded.getCard()->getName() << ")";
// << " card: " << cardBeingDownloaded.getCard()->getName() << ")";
mutex.lock(); mutex.lock();
loadQueue.prepend(cardBeingDownloaded); loadQueue.prepend(cardBeingDownloaded);
mutex.unlock(); mutex.unlock();
} else { } else {
qDebug() << "Picture NOT found, download failed, no more url combinations to try: BAILING OUT for card: " qDebug() << "Picture NOT found, download failed, no more sets to try: BAILING OUT (oldset: "
<< cardBeingDownloaded.getCard()->getName() << ")"; << cardBeingDownloaded.getSetName() << " card: " << cardBeingDownloaded.getCard()->getName() << ")";
imageLoaded(cardBeingDownloaded.getCard(), QImage()); imageLoaded(cardBeingDownloaded.getCard(), QImage());
cardBeingDownloaded.clear(); cardBeingDownloaded.clear();
} }
@ -374,7 +302,7 @@ void PictureLoaderWorker::picDownloadFinished(QNetworkReply *reply)
if (statusCode == 301 || statusCode == 302) { if (statusCode == 301 || statusCode == 302) {
QUrl redirectUrl = reply->attribute(QNetworkRequest::RedirectionTargetAttribute).toUrl(); QUrl redirectUrl = reply->attribute(QNetworkRequest::RedirectionTargetAttribute).toUrl();
QNetworkRequest req(redirectUrl); QNetworkRequest req(redirectUrl);
qDebug() << "following redirect:" << cardBeingDownloaded.getCard()->getName() << "Url:" << req.url().toString(); qDebug() << "following redirect:" << cardBeingDownloaded.getCard()->getName() << "Url:" << req.url();
networkManager->get(req); networkManager->get(req);
return; return;
} }
@ -417,9 +345,7 @@ void PictureLoaderWorker::picDownloadFinished(QNetworkReply *reply)
} }
imageLoaded(cardBeingDownloaded.getCard(), testImage); imageLoaded(cardBeingDownloaded.getCard(), testImage);
qDebug() << "Image successfully downloaded from " << reply->request().url().toString();
} else { } else {
qDebug() << "Possible picture at " << reply->request().url().toString() << " could not be loaded";
picDownloadFailed(); picDownloadFailed();
} }

View file

@ -18,10 +18,7 @@ private:
CardInfoPtr card; CardInfoPtr card;
QList<CardSetPtr> sortedSets; QList<CardSetPtr> sortedSets;
QList<QString> urlTemplates;
QList<QString> currentSetUrls;
int setIndex; int setIndex;
int urlIndex;
public: public:
PictureToLoad(CardInfoPtr _card = CardInfoPtr()); PictureToLoad(CardInfoPtr _card = CardInfoPtr());
@ -34,12 +31,8 @@ public:
card.clear(); card.clear();
} }
CardSetPtr getCurrentSet() const; CardSetPtr getCurrentSet() const;
QString getCurrentUrl() const;
QString getSetName() const; QString getSetName() const;
QString transformUrl(QString urlTemplate) const;
bool nextSet(); bool nextSet();
bool nextUrl();
void populateSetUrls();
}; };
class PictureLoaderWorker : public QObject class PictureLoaderWorker : public QObject
@ -64,7 +57,7 @@ private:
PictureToLoad cardBeingDownloaded; PictureToLoad cardBeingDownloaded;
bool picDownload, downloadRunning, loadQueueRunning; bool picDownload, downloadRunning, loadQueueRunning;
void startNextPicDownload(); void startNextPicDownload();
QList<QString> getAllPicUrls(); QString getPicUrl();
bool cardImageExistsOnDisk(QString &setName, QString &correctedCardname); bool cardImageExistsOnDisk(QString &setName, QString &correctedCardname);
bool imageIsBlackListed(const QByteArray &picData); bool imageIsBlackListed(const QByteArray &picData);
private slots: private slots: