From f3a57d5506cf4ea9a680c4f2bd8a48e32348295b Mon Sep 17 00:00:00 2001 From: Fabio Bas Date: Mon, 16 Jun 2014 00:27:33 +0200 Subject: [PATCH] Import data from mtgjson's allsets.xml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add multiverse id to cards.xml * remove pictures urls from cards.xml TBD: * fix/clean oracle’s gui * add support in cockatrice --- cockatrice/src/carddatabase.cpp | 23 +- cockatrice/src/carddatabase.h | 3 + oracle/src/oracleimporter.cpp | 395 +++++++++++++------------------- oracle/src/oracleimporter.h | 32 +-- oracle/src/window_main.cpp | 8 +- 5 files changed, 193 insertions(+), 268 deletions(-) diff --git a/cockatrice/src/carddatabase.cpp b/cockatrice/src/carddatabase.cpp index e5c60e52..748d8197 100644 --- a/cockatrice/src/carddatabase.cpp +++ b/cockatrice/src/carddatabase.cpp @@ -406,12 +406,27 @@ QXmlStreamWriter &operator<<(QXmlStreamWriter &xml, const CardInfo *info) xml.writeTextElement("name", info->getName()); const SetList &sets = info->getSets(); + QString tmpString; + QString tmpSet; for (int i = 0; i < sets.size(); i++) { xml.writeStartElement("set"); - xml.writeAttribute("picURL", info->getPicURL(sets[i]->getShortName())); - xml.writeAttribute("picURLHq", info->getPicURLHq(sets[i]->getShortName())); - xml.writeAttribute("picURLSt", info->getPicURLSt(sets[i]->getShortName())); - xml.writeCharacters(sets[i]->getShortName()); + + tmpSet=sets[i]->getShortName(); + xml.writeAttribute("muId", QString::number(info->getMuId(tmpSet))); + + tmpString = info->getPicURL(tmpSet); + if(!tmpString.isEmpty()) + xml.writeAttribute("picURL", tmpString); + + tmpString = info->getPicURLHq(tmpSet); + if(!tmpString.isEmpty()) + xml.writeAttribute("picURLHq", tmpString); + + tmpString = info->getPicURLSt(tmpSet); + if(!tmpString.isEmpty()) + xml.writeAttribute("picURLSt", tmpString); + + xml.writeCharacters(tmpSet); xml.writeEndElement(); } const QStringList &colors = info->getColors(); diff --git a/cockatrice/src/carddatabase.h b/cockatrice/src/carddatabase.h index 9d3f33f3..987848d3 100644 --- a/cockatrice/src/carddatabase.h +++ b/cockatrice/src/carddatabase.h @@ -100,6 +100,7 @@ private: QStringList colors; int loyalty; QMap picURLs, picURLsHq, picURLsSt; + QMap muIds; bool cipt; int tableRow; QPixmap *pixmap; @@ -139,6 +140,7 @@ public: QString getPicURL(const QString &set) const { return picURLs.value(set); } QString getPicURLHq(const QString &set) const { return picURLsHq.value(set); } QString getPicURLSt(const QString &set) const { return picURLsSt.value(set); } + int getMuId(const QString &set) const { return muIds.value(set); } QString getPicURL() const; const QMap &getPicURLs() const { return picURLs; } QString getMainCardType() const; @@ -149,6 +151,7 @@ public: void setPicURL(const QString &_set, const QString &_picURL) { picURLs.insert(_set, _picURL); } void setPicURLHq(const QString &_set, const QString &_picURL) { picURLsHq.insert(_set, _picURL); } void setPicURLSt(const QString &_set, const QString &_picURL) { picURLsSt.insert(_set, _picURL); } + void setMuId(const QString &_set, const int &_muId) { muIds.insert(_set, _muId); } void addToSet(CardSet *set); QPixmap *loadPixmap(); QPixmap *getPixmap(QSize size); diff --git a/oracle/src/oracleimporter.cpp b/oracle/src/oracleimporter.cpp index 4adfb1a8..b3aa5e44 100644 --- a/oracle/src/oracleimporter.cpp +++ b/oracle/src/oracleimporter.cpp @@ -1,162 +1,138 @@ #include "oracleimporter.h" #include -#include -#include -#include #include #include "qt-json/json.h" OracleImporter::OracleImporter(const QString &_dataDir, QObject *parent) - : CardDatabase(parent), dataDir(_dataDir), setIndex(-1) + : CardDatabase(parent), dataDir(_dataDir) { - buffer = new QBuffer(this); - http = new QHttp(this); - connect(http, SIGNAL(requestFinished(int, bool)), this, SLOT(httpRequestFinished(int, bool))); - connect(http, SIGNAL(responseHeaderReceived(const QHttpResponseHeader &)), this, SLOT(readResponseHeader(const QHttpResponseHeader &))); - connect(http, SIGNAL(dataReadProgress(int, int)), this, SIGNAL(dataReadProgress(int, int))); } bool OracleImporter::readSetsFromFile(const QString &fileName) { - QFile setsFile(fileName); - if (!setsFile.open(QIODevice::ReadOnly | QIODevice::Text)) { - QMessageBox::critical(0, tr("Error"), tr("Cannot open file '%1'.").arg(fileName)); - return false; - } + QFile setsFile(fileName); + if (!setsFile.open(QIODevice::ReadOnly | QIODevice::Text)) { + QMessageBox::critical(0, tr("Error"), tr("Cannot open file '%1'.").arg(fileName)); + return false; + } - QXmlStreamReader xml(&setsFile); - return readSetsFromXml(xml); + return readSetsFromByteArray(setsFile.readAll()); } bool OracleImporter::readSetsFromByteArray(const QByteArray &data) { - QXmlStreamReader xml(data); - return readSetsFromXml(xml); -} - -bool OracleImporter::readSetsFromXml(QXmlStreamReader &xml) -{ - QList newSetList; - - QString edition; - QString editionLong; - QString editionURL; - while (!xml.atEnd()) { - if (xml.readNext() == QXmlStreamReader::EndElement) - break; - if (xml.name() == "set") { - bool import = xml.attributes().value("import").toString().toInt(); - while (!xml.atEnd()) { - if (xml.readNext() == QXmlStreamReader::EndElement) - break; - if (xml.name() == "name") - edition = xml.readElementText(); - else if (xml.name() == "longname") - editionLong = xml.readElementText(); - else if (xml.name() == "url") - editionURL = xml.readElementText(); - } - newSetList.append(SetToDownload(edition, editionLong, editionURL, import)); - edition = editionLong = editionURL = QString(); - } else if (xml.name() == "picture_url") - pictureUrl = xml.readElementText(); - else if (xml.name() == "picture_url_hq") - pictureUrlHq = xml.readElementText(); - else if (xml.name() == "picture_url_st") - pictureUrlSt = xml.readElementText(); - else if (xml.name() == "set_url") - setUrl = xml.readElementText(); - } - if (newSetList.isEmpty()) - return false; - allSets = newSetList; - return true; -} - -CardInfo *OracleImporter::addCard(const QString &setName, - QString cardName, - bool isToken, - int cardId, - const QString &cardCost, - const QString &cardType, - const QString &cardPT, - int cardLoyalty, - const QStringList &cardText) -{ - QString fullCardText = cardText.join("\n"); - bool splitCard = false; - if (cardName.contains('(')) { - cardName.remove(QRegExp(" \\(.*\\)")); - splitCard = true; - } - // Workaround for card name weirdness - if (cardName.contains("XX")) - cardName.remove("XX"); - cardName = cardName.replace("Æ", "AE"); - cardName = cardName.replace("’", "'"); - - CardInfo *card; - if (cardHash.contains(cardName)) { - card = cardHash.value(cardName); - if (splitCard && !card->getText().contains(fullCardText)) - card->setText(card->getText() + "\n---\n" + fullCardText); - } else { - bool mArtifact = false; - if (cardType.endsWith("Artifact")) - for (int i = 0; i < cardText.size(); ++i) - if (cardText[i].contains("{T}") && cardText[i].contains("to your mana pool")) - mArtifact = true; - - QStringList colors; - QStringList allColors = QStringList() << "W" << "U" << "B" << "R" << "G"; - for (int i = 0; i < allColors.size(); i++) - if (cardCost.contains(allColors[i])) - colors << allColors[i]; - - if (cardText.contains(cardName + " is white.")) - colors << "W"; - if (cardText.contains(cardName + " is blue.")) - colors << "U"; - if (cardText.contains(cardName + " is black.")) - colors << "B"; - if (cardText.contains(cardName + " is red.")) - colors << "R"; - if (cardText.contains(cardName + " is green.")) - colors << "G"; - - bool cipt = (cardText.contains(cardName + " enters the battlefield tapped.")); - - card = new CardInfo(this, cardName, isToken, cardCost, cardType, cardPT, fullCardText, colors, cardLoyalty, cipt); - int tableRow = 1; - QString mainCardType = card->getMainCardType(); - if ((mainCardType == "Land") || mArtifact) - tableRow = 0; - else if ((mainCardType == "Sorcery") || (mainCardType == "Instant")) - tableRow = 3; - else if (mainCardType == "Creature") - tableRow = 2; - card->setTableRow(tableRow); - - cardHash.insert(cardName, card); - } - card->setPicURL(setName, getPictureUrl(pictureUrl, cardId, cardName, setName)); - card->setPicURLHq(setName, getPictureUrl(pictureUrlHq, cardId, cardName, setName)); - card->setPicURLSt(setName, getPictureUrl(pictureUrlSt, cardId, cardName, setName)); - return card; -} - -int OracleImporter::importTextSpoiler(CardSet *set, const QByteArray &data) -{ - int cards = 0; + QList newSetList; + bool ok; - QVariantMap resultMap = QtJson::Json::parse(QString(data), ok).toMap(); + setsMap = QtJson::Json::parse(QString(data), ok).toMap(); if (!ok) { qDebug() << "error: QtJson::Json::parse()"; return 0; } - QListIterator it(resultMap.value("cards").toList()); + QListIterator it(setsMap.values()); + QVariantMap map; + + QString edition; + QString editionLong; + QVariant editionCards; + bool import; + + while (it.hasNext()) { + map = it.next().toMap(); + edition = map.value("code").toString(); + editionLong = map.value("name").toString(); + editionCards = map.value("cards"); + + // core and expansion sets are marked to be imported by default + import = (0 == QString::compare(map.value("type").toString(), QString("core"), Qt::CaseInsensitive) || + 0 == QString::compare(map.value("type").toString(), QString("expansion"), Qt::CaseInsensitive)); + + newSetList.append(SetToDownload(edition, editionLong, editionCards, import)); + } + if (newSetList.isEmpty()) + return false; + allSets = newSetList; + return true; +} + +CardInfo *OracleImporter::addCard(const QString &setName, + QString cardName, + bool isToken, + int cardId, + const QString &cardCost, + const QString &cardType, + const QString &cardPT, + int cardLoyalty, + const QStringList &cardText) +{ + QString fullCardText = cardText.join("\n"); + bool splitCard = false; + if (cardName.contains('(')) { + cardName.remove(QRegExp(" \\(.*\\)")); + splitCard = true; + } + // Workaround for card name weirdness + if (cardName.contains("XX")) + cardName.remove("XX"); + cardName = cardName.replace("Æ", "AE"); + cardName = cardName.replace("’", "'"); + + CardInfo *card; + if (cardHash.contains(cardName)) { + card = cardHash.value(cardName); + if (splitCard && !card->getText().contains(fullCardText)) + card->setText(card->getText() + "\n---\n" + fullCardText); + } else { + bool mArtifact = false; + if (cardType.endsWith("Artifact")) + for (int i = 0; i < cardText.size(); ++i) + if (cardText[i].contains("{T}") && cardText[i].contains("to your mana pool")) + mArtifact = true; + + QStringList colors; + QStringList allColors = QStringList() << "W" << "U" << "B" << "R" << "G"; + for (int i = 0; i < allColors.size(); i++) + if (cardCost.contains(allColors[i])) + colors << allColors[i]; + + if (cardText.contains(cardName + " is white.")) + colors << "W"; + if (cardText.contains(cardName + " is blue.")) + colors << "U"; + if (cardText.contains(cardName + " is black.")) + colors << "B"; + if (cardText.contains(cardName + " is red.")) + colors << "R"; + if (cardText.contains(cardName + " is green.")) + colors << "G"; + + bool cipt = (cardText.contains(cardName + " enters the battlefield tapped.")); + + card = new CardInfo(this, cardName, isToken, cardCost, cardType, cardPT, fullCardText, colors, cardLoyalty, cipt); + int tableRow = 1; + QString mainCardType = card->getMainCardType(); + if ((mainCardType == "Land") || mArtifact) + tableRow = 0; + else if ((mainCardType == "Sorcery") || (mainCardType == "Instant")) + tableRow = 3; + else if (mainCardType == "Creature") + tableRow = 2; + card->setTableRow(tableRow); + + cardHash.insert(cardName, card); + } + card->setMuId(setName, cardId); + + return card; +} + +int OracleImporter::importTextSpoiler(CardSet *set, const QVariant &data) +{ + int cards = 0; + + QListIterator it(data.toList()); QVariantMap map; QString cardName; QString cardCost; @@ -237,111 +213,52 @@ int OracleImporter::importTextSpoiler(CardSet *set, const QByteArray &data) return cards; } - +/* QString OracleImporter::getPictureUrl(QString url, int cardId, QString name, const QString &setName) const { - if ((name == "Island") || (name == "Swamp") || (name == "Mountain") || (name == "Plains") || (name == "Forest")) - name.append("1"); - return url.replace("!cardid!", QString::number(cardId)).replace("!set!", setName).replace("!name!", name - .replace("ö", "o") -// .remove('\'') - .remove(" // ") -// .remove(',') -// .remove(':') -// .remove('.') - .remove(QRegExp("\\(.*\\)")) - .simplified() -// .replace(' ', '_') -// .replace('-', '_') - ); + if ((name == "Island") || (name == "Swamp") || (name == "Mountain") || (name == "Plains") || (name == "Forest")) + name.append("1"); + return url.replace("!cardid!", QString::number(cardId)).replace("!set!", setName).replace("!name!", name + .replace("ö", "o") +// .remove('\'') + .remove(" // ") +// .remove(',') +// .remove(':') +// .remove('.') + .remove(QRegExp("\\(.*\\)")) + .simplified() +// .replace(' ', '_') +// .replace('-', '_') + ); } - -int OracleImporter::startDownload() +*/ +int OracleImporter::startImport() { - clear(); - - setsToDownload.clear(); - for (int i = 0; i < allSets.size(); ++i) - if (allSets[i].getImport()) - setsToDownload.append(allSets[i]); - - if (setsToDownload.isEmpty()) - return 0; - setIndex = 0; - emit setIndexChanged(0, 0, setsToDownload[0].getLongName()); - - downloadNextFile(); - return setsToDownload.size(); -} - -void OracleImporter::downloadNextFile() -{ - QString urlString = setsToDownload[setIndex].getUrl(); - if (urlString.isEmpty()) - urlString = setUrl; - urlString = urlString.replace("!name!", setsToDownload[setIndex].getShortName()); - - if (urlString.startsWith("http://")) { - QUrl url(urlString); - http->setHost(url.host(), QHttp::ConnectionModeHttp, url.port() == -1 ? 0 : url.port()); - QString path = QUrl::toPercentEncoding(urlString.mid(url.host().size() + 7).replace(' ', '+'), "?!$&'()*+,;=:@/"); - - buffer->close(); - buffer->setData(QByteArray()); - buffer->open(QIODevice::ReadWrite | QIODevice::Text); - reqId = http->get(path, buffer); - } else { - QFile file(dataDir + "/" + urlString); - file.open(QIODevice::ReadOnly | QIODevice::Text); - - buffer->close(); - buffer->setData(file.readAll()); - buffer->open(QIODevice::ReadWrite | QIODevice::Text); - reqId = 0; - httpRequestFinished(reqId, false); - } -} - -void OracleImporter::httpRequestFinished(int requestId, bool error) -{ - if (error) { - QMessageBox::information(0, tr("HTTP"), tr("Error.")); - return; - } - if (requestId != reqId) - return; - - CardSet *set = new CardSet(setsToDownload[setIndex].getShortName(), setsToDownload[setIndex].getLongName()); - if (!setHash.contains(set->getShortName())) - setHash.insert(set->getShortName(), set); - - buffer->seek(0); - buffer->close(); - int cards = importTextSpoiler(set, buffer->data()); - if (cards > 0) - ++setIndex; - - if (setIndex == setsToDownload.size()) { - emit setIndexChanged(cards, setIndex, QString()); - setIndex = -1; - } else { - downloadNextFile(); - emit setIndexChanged(cards, setIndex, setsToDownload[setIndex].getLongName()); - } -} - -void OracleImporter::readResponseHeader(const QHttpResponseHeader &responseHeader) -{ - switch (responseHeader.statusCode()) { - case 200: - case 301: - case 302: - case 303: - case 307: - break; - default: - QMessageBox::information(0, tr("HTTP"), tr("Download failed: %1.").arg(responseHeader.reasonPhrase())); - http->abort(); - deleteLater(); - } + clear(); + + int setCards = 0, setIndex= 0; + QListIterator it(allSets); + const SetToDownload * curSet; + + while (it.hasNext()) + { + curSet = & it.next(); + + emit setIndexChanged(0, 0, curSet->getLongName()); + + CardSet *set = new CardSet(curSet->getShortName(), curSet->getLongName()); + if (!setHash.contains(set->getShortName())) + setHash.insert(set->getShortName(), set); + + int setCards = importTextSpoiler(set, curSet->getCards()); + + ++setIndex; + + emit setIndexChanged(setCards, setIndex, curSet->getLongName()); + } + + emit setIndexChanged(setCards, setIndex, QString()); + + // total number of sets + return setIndex; } diff --git a/oracle/src/oracleimporter.h b/oracle/src/oracleimporter.h index 3e03586f..76989933 100644 --- a/oracle/src/oracleimporter.h +++ b/oracle/src/oracleimporter.h @@ -1,44 +1,34 @@ #ifndef ORACLEIMPORTER_H #define ORACLEIMPORTER_H -#include -#include +#include -class QBuffer; -class QXmlStreamReader; +#include class SetToDownload { private: - QString shortName, longName, url; + QString shortName, longName; bool import; + QVariant cards; public: const QString &getShortName() const { return shortName; } const QString &getLongName() const { return longName; } - const QString &getUrl() const { return url; } + const QVariant &getCards() const { return cards; } bool getImport() const { return import; } void setImport(bool _import) { import = _import; } - SetToDownload(const QString &_shortName, const QString &_longName, const QString &_url, bool _import) - : shortName(_shortName), longName(_longName), url(_url), import(_import) { } + SetToDownload(const QString &_shortName, const QString &_longName, const QVariant &_cards, bool _import) + : shortName(_shortName), longName(_longName), cards(_cards), import(_import) { } }; class OracleImporter : public CardDatabase { Q_OBJECT private: - QList allSets, setsToDownload; - QString pictureUrl, pictureUrlHq, pictureUrlSt, setUrl; + QList allSets; + QVariantMap setsMap; QString dataDir; - int setIndex; - int reqId; - QBuffer *buffer; - QHttp *http; - QString getPictureUrl(QString url, int cardId, QString name, const QString &setName) const; void downloadNextFile(); - bool readSetsFromXml(QXmlStreamReader &xml); CardInfo *addCard(const QString &setName, QString cardName, bool isToken, int cardId, const QString &cardCost, const QString &cardType, const QString &cardPT, int cardLoyalty, const QStringList &cardText); -private slots: - void httpRequestFinished(int requestId, bool error); - void readResponseHeader(const QHttpResponseHeader &responseHeader); signals: void setIndexChanged(int cardsImported, int setIndex, const QString &nextSetName); void dataReadProgress(int bytesRead, int totalBytes); @@ -46,8 +36,8 @@ public: OracleImporter(const QString &_dataDir, QObject *parent = 0); bool readSetsFromByteArray(const QByteArray &data); bool readSetsFromFile(const QString &fileName); - int startDownload(); - int importTextSpoiler(CardSet *set, const QByteArray &data); + int startImport(); + int importTextSpoiler(CardSet *set, const QVariant &data); QList &getSets() { return allSets; } const QString &getDataDir() const { return dataDir; } }; diff --git a/oracle/src/window_main.cpp b/oracle/src/window_main.cpp index 0283bded..24d79b4f 100644 --- a/oracle/src/window_main.cpp +++ b/oracle/src/window_main.cpp @@ -19,7 +19,7 @@ #include "window_main.h" #include "oracleimporter.h" -const QString WindowMain::defaultSetsUrl = QString("http://www.woogerworks.com/files/sets.xml"); +const QString WindowMain::defaultSetsUrl = QString("http://mtgjson.com/json/AllSets.json"); WindowMain::WindowMain(QWidget *parent) : QMainWindow(parent) @@ -45,7 +45,7 @@ WindowMain::WindowMain(QWidget *parent) checkAllButtonLayout->addWidget(checkAllButton); checkAllButtonLayout->addWidget(uncheckAllButton); - startButton = new QPushButton(tr("&Start download")); + startButton = new QPushButton(tr("&Start import")); connect(startButton, SIGNAL(clicked()), this, SLOT(actStart())); QVBoxLayout *settingsLayout = new QVBoxLayout; @@ -130,7 +130,7 @@ void WindowMain::actLoadSetsFile() { QFileDialog dialog(this, tr("Load sets file")); dialog.setFileMode(QFileDialog::ExistingFile); - dialog.setNameFilter("Sets XML file (*.xml)"); + dialog.setNameFilter("Sets JSON file (*.json)"); if (!dialog.exec()) return; @@ -224,7 +224,7 @@ void WindowMain::actUncheckAll() void WindowMain::actStart() { - int setsCount = importer->startDownload(); + int setsCount = importer->startImport(); if (!setsCount) { QMessageBox::critical(this, tr("Error"), tr("No sets to download selected.")); return;