diff --git a/app/app.pro b/app/app.pro index 773237a..ac8f281 100644 --- a/app/app.pro +++ b/app/app.pro @@ -6,6 +6,7 @@ TARGET = $$MEDIAWRITER_NAME QT += qml quick widgets network +LIBS += -lisomd5 linux { LIBS += -lyaml-cpp } diff --git a/app/dialogs/DownloadDialog.qml b/app/dialogs/DownloadDialog.qml index 43c3f9b..0605ac0 100644 --- a/app/dialogs/DownloadDialog.qml +++ b/app/dialogs/DownloadDialog.qml @@ -195,6 +195,32 @@ Dialog { } } }, + State { + name: "write_verifying_failed_no_drives" + when: releases.selected.variant.status === Variant.WRITE_VERIFYING_FAILED && drives.length <= 0 + PropertyChanges { + target: rightButton; + text: qsTr("Retry"); + enabled: false; + color: "red"; + onClicked: drives.selected.write(releases.selected.variant); + } + }, + State { + name: "write_verifying_failed" + when: releases.selected.variant.status === Variant.WRITE_VERIFYING_FAILED && drives.length > 0 + PropertyChanges { + target: messageLoseData; + visible: true; + } + PropertyChanges { + target: rightButton; + text: qsTr("Retry"); + enabled: true; + color: "red"; + onClicked: drives.selected.write(releases.selected.variant); + } + }, State { name: "failed_download" when: releases.selected.variant.status === Variant.DOWNLOAD_FAILED @@ -446,6 +472,22 @@ Dialog { text: qsTr("Writing this image type is not supported.") color: "red" } + Text { + visible: releases.selected.variant.noMd5sum + font.pointSize: 10 + Layout.fillWidth: true + width: Layout.width + wrapMode: Text.WordWrap + text: qsTr("This image won't be verified after writing because no MD5 sum was found.") + } + Text { + visible: releases.selected.variant.isCompressed + font.pointSize: 10 + Layout.fillWidth: true + width: Layout.width + wrapMode: Text.WordWrap + text: qsTr("This image won't be verified after writing because it is a compressed image.") + } RowLayout { height: rightButton.height Layout.minimumWidth: parent.width diff --git a/app/image_download.cpp b/app/image_download.cpp index 74a51cb..19d5882 100644 --- a/app/image_download.cpp +++ b/app/image_download.cpp @@ -30,11 +30,12 @@ #include #include -ImageDownload::ImageDownload(const QUrl &url_arg, const QString &filePath_arg) +ImageDownload::ImageDownload(const QUrl &url_arg, const QString &filePath_arg, const QString &md5sum_arg) : QObject() , hash(QCryptographicHash::Md5) { url = url_arg; filePath = filePath_arg; + md5sum = md5sum_arg; file = nullptr; startingImageDownload = false; wasCancelled = false; @@ -122,66 +123,13 @@ void ImageDownload::onImageDownloadFinished() { } else if (reply->error() == QNetworkReply::NoError) { qDebug() << this->metaObject()->className() << "Finished successfully"; - qDebug() << this->metaObject()->className() << "Downloading md5"; + if (md5sum.isEmpty()) { + // If md5sum doesn't exist, be lenient and + // don't treat this as a failed check. + // Instead, skip the check. + qDebug() << this->metaObject()->className() << "No md5sum found, so skipping md5 check"; - const QString md5sumUrl = url.adjusted(QUrl::RemoveFilename).toString() + "/MD5SUM"; - QNetworkReply *md5Reply = makeNetworkRequest(md5sumUrl); - - connect( - md5Reply, &QNetworkReply::finished, - this, &ImageDownload::onMd5DownloadFinished); - connect( - this, &ImageDownload::cancelled, - md5Reply, &QNetworkReply::abort); - } else { - qDebug() << "Download was interrupted by an error:" << reply->errorString(); - qDebug() << "Attempting to resume"; - - emit interrupted(); - - QTimer::singleShot(1000, this, - [this]() { - startImageDownload(); - }); - } -} - -void ImageDownload::onMd5DownloadFinished() { - QNetworkReply *reply = qobject_cast(sender()); - reply->deleteLater(); - - if (wasCancelled) { - return; - } else { - if (reply->error() == QNetworkReply::NoError) { - qDebug() << this->metaObject()->className() << "Downloaded MD5SUM successfully"; - - md5 = [this, reply]() { - const QByteArray md5sumBytes = reply->readAll(); - const QString md5sumContents(md5sumBytes); - - // MD5SUM is of the form "sum image \n sum image \n ..." - // Search for the sum by finding image matching url - const QStringList elements = md5sumContents.split(QRegExp("\\s+")); - QString prev = ""; - for (int i = 0; i < elements.size(); ++i) { - if (elements[i].size() > 0 && url.toString().contains(elements[i]) && prev.size() > 0) { - return prev; - } - - prev = elements[i]; - } - - return QString(); - }(); - } else { - qDebug() << this->metaObject()->className() << "Failed to download MD5SUM"; - - md5 = QString(); - } - - if (md5.isEmpty()) { - checkMd5(QString()); + rename_to_final_name(); } else { file->close(); const bool open_success = file->open(QIODevice::ReadOnly); @@ -194,6 +142,16 @@ void ImageDownload::onMd5DownloadFinished() { finish(ImageDownload::Md5CheckFail); } } + } else { + qDebug() << "Download was interrupted by an error:" << reply->errorString(); + qDebug() << "Attempting to resume"; + + emit interrupted(); + + QTimer::singleShot(1000, this, + [this]() { + startImageDownload(); + }); } } @@ -212,7 +170,20 @@ void ImageDownload::computeMd5() { if (file->atEnd()) { const QByteArray sum_bytes = hash.result().toHex(); const QString computedMd5 = QString(sum_bytes); - checkMd5(computedMd5); + + const bool checkPassed = (computedMd5 == md5sum); + + if (checkPassed) { + qDebug() << "MD5 check passed"; + + rename_to_final_name(); + } else { + qDebug() << "MD5 mismatch"; + qDebug() << "sum should be =" << md5sum; + qDebug() << "computed sum =" << computedMd5; + + finish(ImageDownload::Md5CheckFail); + } } else { QTimer::singleShot(0, this, &ImageDownload::computeMd5); } @@ -246,39 +217,15 @@ void ImageDownload::startImageDownload() { reply, &QNetworkReply::abort); } -void ImageDownload::checkMd5(const QString &computedMd5) { - const bool checkPassed = [this, computedMd5]() { - if (md5.isEmpty()) { - // Can fail to download md5 sum if: - // 1) Failed to download MD5SUM file - // 2) MD5SUM file is not present - // 3) MD5SUM file does not contain needed sum - // In all cases, DON'T treat this as a fail. - // Instead, skip the check. - qDebug() << this->metaObject()->className() << "Failed to download md5 sum, so skipping md5 check"; +void ImageDownload::rename_to_final_name() { + qDebug() << this->metaObject()->className() << "Renaming to final filename"; - return true; - } else { - return (computedMd5 == md5); - } - }(); + const bool rename_success = file->rename(filePath); - if (checkPassed) { - qDebug() << this->metaObject()->className() << "Renaming to final filename"; - - const bool rename_success = file->rename(filePath); - - if (rename_success) { - finish(ImageDownload::Success); - } else { - finish(ImageDownload::DiskError, tr("Unable to rename the temporary file.")); - } + if (rename_success) { + finish(ImageDownload::Success); } else { - qDebug() << "MD5 mismatch"; - qDebug() << "sum should be =" << md5; - qDebug() << "computed sum =" << computedMd5; - - finish(ImageDownload::Md5CheckFail); + finish(ImageDownload::DiskError, tr("Unable to rename the temporary file.")); } } diff --git a/app/image_download.h b/app/image_download.h index 9d1df31..684722f 100644 --- a/app/image_download.h +++ b/app/image_download.h @@ -54,7 +54,7 @@ public: Cancelled }; - ImageDownload(const QUrl &url_arg, const QString &filePath_arg); + ImageDownload(const QUrl &url_arg, const QString &filePath_arg, const QString &md5sum_arg); Result result() const; QString errorString() const; @@ -86,7 +86,6 @@ public slots: private slots: void onImageDownloadReadyRead(); void onImageDownloadFinished(); - void onMd5DownloadFinished(); void computeMd5(); private: @@ -95,16 +94,15 @@ private: QUrl url; QString filePath; + QString md5sum; QFile *file; bool startingImageDownload; bool wasCancelled; QCryptographicHash hash; - QString md5; - QString getFilePath() const; void startImageDownload(); - void checkMd5(const QString &computedMd5); + void rename_to_final_name(); void finish(const Result result_arg, const QString &errorString_arg = QString()); }; diff --git a/app/linuxdrivemanager.cpp b/app/linuxdrivemanager.cpp index 9ffaf99..195f675 100644 --- a/app/linuxdrivemanager.cpp +++ b/app/linuxdrivemanager.cpp @@ -220,6 +220,7 @@ bool LinuxDrive::write(Variant *variant) { args << "write"; args << variant->filePath(); args << m_device; + args << variant->md5sum(); qDebug() << this->metaObject()->className() << "Helper command will be" << args; m_process->setArguments(args); @@ -286,7 +287,9 @@ void LinuxDrive::onReadyRead() { m_progress->setCurrent(NAN); - m_variant->setStatus(Variant::WRITING); + if (m_variant->status() != Variant::WRITE_VERIFYING && m_variant->status() != Variant::WRITING) { + m_variant->setStatus(Variant::WRITING); + } while (m_process->bytesAvailable() > 0) { QString line = m_process->readLine().trimmed(); @@ -297,6 +300,14 @@ void LinuxDrive::onReadyRead() { m_progress->setMax(file.size()); m_progress->setCurrent(0); + + m_variant->setStatus(Variant::WRITING); + } else if (line == "CHECK") { + qDebug() << this->metaObject()->className() << "Helper finished writing, now it will check the written data"; + const QFile file(m_variant->filePath()); + m_progress->setMax(file.size()); + m_progress->setCurrent(0); + m_variant->setStatus(Variant::WRITE_VERIFYING); } else if (line == "DONE") { m_variant->setStatus(Variant::WRITING_FINISHED); Notifications::notify(tr("Finished!"), tr("Writing %1 was successful").arg(m_variant->fileName())); @@ -321,8 +332,12 @@ void LinuxDrive::onFinished(const int exitCode, const QProcess::ExitStatus statu QString errorMessage = m_process->readAllStandardError(); qDebug() << "Writing failed:" << errorMessage; Notifications::notify(tr("Error"), tr("Writing %1 failed").arg(m_variant->fileName())); - if (m_variant->status() == Variant::WRITING) { - m_variant->setErrorString(errorMessage); + + m_variant->setErrorString(errorMessage); + + if (m_variant->status() == Variant::WRITE_VERIFYING) { + m_variant->setStatus(Variant::WRITE_VERIFYING_FAILED); + } else { m_variant->setStatus(Variant::WRITING_FAILED); } } else { diff --git a/app/releasemanager.cpp b/app/releasemanager.cpp index bd98a15..68446de 100644 --- a/app/releasemanager.cpp +++ b/app/releasemanager.cpp @@ -47,6 +47,7 @@ ReleaseManager::ReleaseManager(QObject *parent) metadata_urls_reply_group = nullptr; metadata_urls_backup_reply_group = nullptr; metadata_reply_group = nullptr; + md5sum_reply_group = nullptr; qDebug() << this->metaObject()->className() << "construction"; @@ -319,7 +320,7 @@ void ReleaseManager::onMetadataDownloaded() { loadReleases(sectionsFiles); - const QList imagesFiles = [&]() { + imagesFiles = [&]() { QList out; for (const QString &image_url : image_urls) { @@ -329,15 +330,148 @@ void ReleaseManager::onMetadataDownloaded() { return out; }(); - qDebug() << "Loading variants"; - - for (const QString &imagesFile : imagesFiles) { - loadVariants(imagesFile); - } + // NOTE: images/variants are loaded later after + // md5sum is downloaded delete metadata_reply_group; metadata_reply_group = nullptr; + const QList md5sum_url_list = [&]() { + const QList image_url_list = [&]() { + QList out; + + for (const QString &imagesFile : imagesFiles) { + YAML::Node variants = YAML::Load(imagesFile.toStdString()); + + if (!variants["entries"]) { + continue; + } + + for (const YAML::Node &variantData : variants["entries"]) { + const QString url = yml_get(variantData, "link"); + out.append(url); + } + } + + return out; + }(); + + const QList out = [&]() { + // NOTE: using set because there will be + // duplicates due to there being multiple + // images per folder + QSet out_set; + + for (const QString &image_url : image_url_list) { + // TODO: duplicating code in + // image_download.cpp + const QString md5sum_url = QUrl(image_url).adjusted(QUrl::RemoveFilename).toString() + "/MD5SUM"; + + out_set.insert(md5sum_url); + } + + const QList out = out_set.toList(); + + return out; + }(); + + return out; + }(); + + downloadMD5SUM(md5sum_url_list); +} + +void ReleaseManager::downloadMD5SUM(const QList &md5sum_url_list) { + qDebug() << "Downloading MD5SUM's"; + + md5sum_reply_group = new NetworkReplyGroup(md5sum_url_list, this); + + connect( + md5sum_reply_group, &NetworkReplyGroup::finished, + this, &ReleaseManager::onMD5SUMDownloaded); +} + +void ReleaseManager::onMD5SUMDownloaded() { + const QHash replies = md5sum_reply_group->get_reply_list(); + + // Check that all replies suceeded + // If not, retry + // TODO: duplicating code + for (const QNetworkReply *reply : replies.values()) { + // NOTE: ignore ContentNotFoundError for + // metadata since it can happen if one of + // the files was moved or renamed. In that + // case it's fine to process other + // downloads and ignore this failed one. + const QNetworkReply::NetworkError error = reply->error(); + const bool download_failed = (error != QNetworkReply::NoError && error != QNetworkReply::ContentNotFoundError); + + if (download_failed) { + qDebug() << "Failed to download md5sum:" << reply->errorString() << reply->error() << "Retrying in 10 seconds."; + QTimer::singleShot(10000, this, &ReleaseManager::downloadMetadata); + + delete md5sum_reply_group; + md5sum_reply_group = nullptr; + + return; + } + } + + qDebug() << "Downloaded md5sum, loading it"; + + const QList md5sum_file_list = [&]() { + QList out; + + for (const QString &url : replies.keys()) { + QNetworkReply *reply = replies[url]; + + if (reply->error() == QNetworkReply::NoError) { + const QByteArray bytes = reply->readAll(); + const QString string = QString(bytes); + out.append(string); + } else { + qDebug() << "Failed to download metadata from" << url; + qDebug() << "Error:" << reply->error(); + } + } + + return out; + }(); + + const QHash md5sum_map = [&]() { + QHash out; + + for (const QString &file : md5sum_file_list) { + const QList line_list = file.split("\n"); + + // MD5SUM is of the form "sum image \n sum + // image \n ..." + for (const QString &line : line_list) { + const QList elements = line.split(QRegExp("\\s+")); + + if (elements.size() != 2) { + continue; + } + + const QString md5sum = elements[0]; + const QString filename = elements[1]; + + out[filename] = md5sum; + } + } + + return out; + }(); + + qDebug() << "Loading variants"; + + for (const QString &imagesFile : imagesFiles) { + loadVariants(imagesFile, md5sum_map); + } + + delete md5sum_reply_group; + md5sum_reply_group = nullptr; + setDownloadingMetadata(false); } @@ -376,7 +510,7 @@ ReleaseFilterModel *ReleaseManager::getFilterModel() const { return filterModel; } -void ReleaseManager::loadVariants(const QString &variantsFile) { +void ReleaseManager::loadVariants(const QString &variantsFile, const QHash &md5sum_map) { YAML::Node variants = YAML::Load(variantsFile.toStdString()); if (!variants["entries"]) { @@ -434,6 +568,13 @@ void ReleaseManager::loadVariants(const QString &variantsFile) { } }(); + const QString md5sum = [&]() { + const QString filename = QUrl(url).fileName(); + const QString out = md5sum_map[filename]; + + return out; + }(); + // qDebug() << QUrl(url).fileName() << releaseName << architecture_name(arch) << board << file_type_name(fileType) << (live ? "LIVE" : ""); // Find a release that has the same name as this variant @@ -449,7 +590,7 @@ void ReleaseManager::loadVariants(const QString &variantsFile) { }(); if (release != nullptr) { - Variant *variant = new Variant(url, arch, fileType, board, live, this); + Variant *variant = new Variant(url, arch, fileType, board, live, md5sum, this); release->addVariant(variant); } else { qDebug() << "Failed to find a release for this variant!" << url; diff --git a/app/releasemanager.h b/app/releasemanager.h index 2370977..6386381 100644 --- a/app/releasemanager.h +++ b/app/releasemanager.h @@ -29,6 +29,7 @@ */ #include +#include class Release; class ReleaseModel; @@ -73,19 +74,23 @@ private: NetworkReplyGroup *metadata_reply_group; NetworkReplyGroup *metadata_urls_reply_group; NetworkReplyGroup *metadata_urls_backup_reply_group; + NetworkReplyGroup *md5sum_reply_group; QList section_urls; QList image_urls; + QList imagesFiles; - void loadVariants(const QString &variantsFile); + void loadVariants(const QString &variantsFile, const QHash &md5sum_map); void setDownloadingMetadata(const bool value); void downloadMetadataUrls(); void onMetadataUrlsDownloaded(); void downloadMetadataUrlsBackup(); void onMetadataUrlsBackupDownloaded(); void downloadMetadata(); + void downloadMD5SUM(const QList &md5sum_url_list); void loadReleases(const QList §ionsFiles); void addReleaseToModel(const int index, Release *release); void onMetadataDownloaded(); + void onMD5SUMDownloaded(); }; #endif // RELEASEMANAGER_H diff --git a/app/variant.cpp b/app/variant.cpp index 9268797..82d72cf 100644 --- a/app/variant.cpp +++ b/app/variant.cpp @@ -33,13 +33,14 @@ #include #include -Variant::Variant(const QString &url, const Architecture arch, const FileType fileType, const QString &board, const bool live, QObject *parent) +Variant::Variant(const QString &url, const Architecture arch, const FileType fileType, const QString &board, const bool live, const QString &md5sum, QObject *parent) : QObject(parent) { m_url = url; m_fileName = QUrl(url).fileName(); m_filePath = QDir(QStandardPaths::writableLocation(QStandardPaths::DownloadLocation)).filePath(fileName()); m_board = board; m_live = live; + m_md5sum = md5sum; m_arch = arch; m_fileType = fileType; m_status = Variant::PREPARING; @@ -53,6 +54,7 @@ Variant::Variant(const QString &path, QObject *parent) m_filePath = path; m_board = QString(); m_live = false; + m_md5sum = QString(); m_arch = Architecture_UNKNOWN; m_fileType = file_type_from_filename(path); m_status = Variant::READY_FOR_WRITING; @@ -71,6 +73,10 @@ QString Variant::fileTypeName() const { return file_type_name(m_fileType); } +QString Variant::md5sum() const { + return m_md5sum; +} + QString Variant::name() const { QString out = architecture_name(m_arch) + " | " + m_board; @@ -93,6 +99,14 @@ bool Variant::canWrite() const { return file_type_can_write(m_fileType); } +bool Variant::noMd5sum() const { + return md5sum().isEmpty(); +} + +bool Variant::isCompressed() const { + return (m_fileType == FileType_TAR_XZ || m_fileType == FileType_IMG_XZ); +} + Progress *Variant::progress() { return m_progress; } @@ -168,7 +182,7 @@ void Variant::download() { setStatus(READY_FOR_WRITING); } else { // Download image - auto download = new ImageDownload(QUrl(url()), filePath()); + auto download = new ImageDownload(QUrl(url()), filePath(), md5sum()); connect( download, &ImageDownload::started, diff --git a/app/variant.h b/app/variant.h index 74fb7e0..56db20f 100644 --- a/app/variant.h +++ b/app/variant.h @@ -59,6 +59,8 @@ class Variant final : public QObject { Q_PROPERTY(QString fileName READ fileName CONSTANT) Q_PROPERTY(QString fileTypeName READ fileTypeName CONSTANT) Q_PROPERTY(bool canWrite READ canWrite CONSTANT) + Q_PROPERTY(bool noMd5sum READ noMd5sum CONSTANT) + Q_PROPERTY(bool isCompressed READ isCompressed CONSTANT) Q_PROPERTY(Progress *progress READ progress CONSTANT) Q_PROPERTY(Status status READ status NOTIFY statusChanged) @@ -76,6 +78,8 @@ public: READY_FOR_WRITING, WRITING, WRITING_FINISHED, + WRITE_VERIFYING, + WRITE_VERIFYING_FAILED, WRITING_FAILED }; Q_ENUMS(Status) @@ -89,10 +93,12 @@ public: {READY_FOR_WRITING, tr("Ready to write")}, {WRITING, tr("Writing")}, {WRITING_FINISHED, tr("Finished!")}, + {WRITE_VERIFYING, tr("Checking the written data")}, + {WRITE_VERIFYING_FAILED, tr("The written data is corrupted")}, {WRITING_FAILED, tr("Error")}, }; - Variant(const QString &url, const Architecture arch, const FileType fileType, const QString &board, const bool live, QObject *parent); + Variant(const QString &url, const Architecture arch, const FileType fileType, const QString &board, const bool live, const QString &md5sum, QObject *parent); // Constructor for local file Variant(const QString &path, QObject *parent); @@ -106,7 +112,10 @@ public: QString filePath() const; QString fileName() const; QString fileTypeName() const; + QString md5sum() const; bool canWrite() const; + bool noMd5sum() const; + bool isCompressed() const; Progress *progress(); Status status() const; @@ -135,6 +144,7 @@ private: QString m_filePath; QString m_board; bool m_live; + QString m_md5sum; Architecture m_arch; FileType m_fileType; Status m_status; diff --git a/app/windrivemanager.cpp b/app/windrivemanager.cpp index e503c8c..5f4eb0d 100644 --- a/app/windrivemanager.cpp +++ b/app/windrivemanager.cpp @@ -291,6 +291,7 @@ bool WinDrive::write(Variant *variant) { args << "write"; args << variant->filePath(); args << QString("%1").arg(m_device); + args << variant->md5sum(); m_child->setArguments(args); qDebug() << this->metaObject()->className() << "Starting" << m_child->program() << args; @@ -360,7 +361,12 @@ void WinDrive::onFinished(const int exitCode, const QProcess::ExitStatus exitSta Notifications::notify(tr("Finished!"), tr("Writing %1 was successful").arg(m_variant->fileName())); } else { m_variant->setErrorString(m_child->readAllStandardError().trimmed()); - m_variant->setStatus(Variant::WRITING_FAILED); + + if (m_variant->status() == Variant::WRITE_VERIFYING) { + m_variant->setStatus(Variant::WRITE_VERIFYING_FAILED); + } else { + m_variant->setStatus(Variant::WRITING_FAILED); + } } m_child->deleteLater(); @@ -393,7 +399,9 @@ void WinDrive::onReadyRead() { m_progress->setCurrent(NAN); - m_variant->setStatus(Variant::WRITING); + if (m_variant->status() != Variant::WRITE_VERIFYING && m_variant->status() != Variant::WRITING) { + m_variant->setStatus(Variant::WRITING); + } while (m_child->bytesAvailable() > 0) { QString line = m_child->readLine().trimmed(); @@ -406,6 +414,12 @@ void WinDrive::onReadyRead() { } else if (line == "DONE") { m_variant->setStatus(Variant::WRITING_FINISHED); Notifications::notify(tr("Finished!"), tr("Writing %1 was successful").arg(m_variant->fileName())); + } else if (line == "CHECK") { + qDebug() << this->metaObject()->className() << "Written media check starting"; + const QFile file(m_variant->filePath()); + m_progress->setMax(file.size()); + m_progress->setCurrent(0); + m_variant->setStatus(Variant::WRITE_VERIFYING); } else { bool ok; qreal bytes = line.toLongLong(&ok); diff --git a/helper/linux/linux.pro b/helper/linux/linux.pro index c406185..f81fcfa 100644 --- a/helper/linux/linux.pro +++ b/helper/linux/linux.pro @@ -5,6 +5,8 @@ QT += core network dbus CONFIG += link_pkgconfig PKGCONFIG += liblzma +LIBS += -lisomd5 + CONFIG += c++11 CONFIG += console diff --git a/helper/linux/main.cpp b/helper/linux/main.cpp index 8cd223e..33a4f3c 100644 --- a/helper/linux/main.cpp +++ b/helper/linux/main.cpp @@ -36,8 +36,8 @@ int main(int argc, char *argv[]) { if (app.arguments().count() == 3 && app.arguments()[1] == "restore") { new RestoreJob(app.arguments()[2]); - } else if (app.arguments().count() == 4 && app.arguments()[1] == "write") { - new WriteJob(app.arguments()[2], app.arguments()[3]); + } else if (app.arguments().count() == 5 && app.arguments()[1] == "write") { + new WriteJob(app.arguments()[2], app.arguments()[3], app.arguments()[4]); } else { QTextStream err(stderr); err << "Helper: Wrong arguments entered"; diff --git a/helper/linux/writejob.cpp b/helper/linux/writejob.cpp index 0ac38ae..339b309 100644 --- a/helper/linux/writejob.cpp +++ b/helper/linux/writejob.cpp @@ -40,6 +40,8 @@ #include +#include "isomd5/libcheckisomd5.h" + typedef QHash Properties; typedef QHash InterfacesAndProperties; typedef QHash DBusIntrospection; @@ -59,10 +61,11 @@ public: size_t size; }; -WriteJob::WriteJob(const QString &what, const QString &where) +WriteJob::WriteJob(const QString &what, const QString &where, const QString &md5_arg) : QObject(nullptr) , what(what) -, where(where) { +, where(where) +, md5(md5_arg) { qDBusRegisterMetaType(); qDBusRegisterMetaType(); qDBusRegisterMetaType(); @@ -75,6 +78,19 @@ WriteJob::WriteJob(const QString &what, const QString &where) QTimer::singleShot(0, this, SLOT(work())); } +int WriteJob::staticOnMediaCheckAdvanced(void *data, long long offset, long long total) { + return ((WriteJob*)data)->onMediaCheckAdvanced(offset, total); +} + +int WriteJob::onMediaCheckAdvanced(long long offset, long long total) { + QTextStream out(stdout); + + Q_UNUSED(total); + out << offset << "\n"; + out.flush(); + return 0; +} + QDBusUnixFileDescriptor WriteJob::getDescriptor() { QTextStream err(stderr); @@ -264,6 +280,55 @@ bool WriteJob::writePlain(int fd) { return true; } +bool WriteJob::check(int fd) { + QTextStream out(stdout); + QTextStream err(stderr); + + if (what.endsWith(".xz")) { + out << "NOT CHECKING BECAUSE IMAGE IS ZIPPED\n"; + out << "DONE\n"; + out.flush(); + err << "OK\n"; + err.flush(); + qApp->exit(0); + return false; + } + + if (md5.isEmpty()) { + out << "NOT CHECKING BECAUSE NO MD5 IS PROVIDED\n"; + out << "DONE\n"; + out.flush(); + err << "OK\n"; + err.flush(); + qApp->exit(0); + return false; + } + + out << "CHECK\n"; + out.flush(); + switch (mediaCheckFD(fd, md5.toLocal8Bit().data(), &WriteJob::staticOnMediaCheckAdvanced, this)) { + case ISOMD5SUM_CHECK_NOT_FOUND: + case ISOMD5SUM_CHECK_PASSED: + out << "DONE\n"; + out.flush(); + err << "OK\n"; + err.flush(); + qApp->exit(0); + return false; + case ISOMD5SUM_CHECK_FAILED: + err << tr("Your drive is probably damaged.") << "\n"; + err.flush(); + qApp->exit(1); + return false; + default: + err << tr("Unexpected error occurred during media check.") << "\n"; + err.flush(); + qApp->exit(1); + return false; + } + return true; +} + void WriteJob::work() { QTextStream out(stdout); QTextStream err(stderr); @@ -293,15 +358,16 @@ void WriteJob::work() { const bool write_success = write(fd.fileDescriptor()); if (write_success) { - out.flush(); - err << "DONE\n"; - qApp->exit(0); + check(fd.fileDescriptor()); } else { qApp->exit(4); } } void WriteJob::onFileChanged(const QString &path) { + QTextStream out(stdout); + QTextStream err(stderr); + const bool still_downloading = QFile::exists(path); if (still_downloading) { return; @@ -313,7 +379,16 @@ void WriteJob::onFileChanged(const QString &path) { return; } - work(); + out << "WRITE\n"; + out.flush(); + + const bool write_success = write(fd.fileDescriptor()); + + if (write_success) { + check(fd.fileDescriptor()); + } else { + qApp->exit(4); + } } PageAlignedBuffer::PageAlignedBuffer(const size_t page_count) { diff --git a/helper/linux/writejob.h b/helper/linux/writejob.h index 7ed876e..ed1c0ad 100644 --- a/helper/linux/writejob.h +++ b/helper/linux/writejob.h @@ -43,12 +43,16 @@ class WriteJob : public QObject { Q_OBJECT public: - explicit WriteJob(const QString &what, const QString &where); + explicit WriteJob(const QString &what, const QString &where, const QString &md5_arg); + + static int staticOnMediaCheckAdvanced(void *data, long long offset, long long total); + int onMediaCheckAdvanced(long long offset, long long total); QDBusUnixFileDescriptor getDescriptor(); bool write(int fd); bool writeCompressed(int fd); bool writePlain(int fd); + bool check(int fd); public slots: void work(); private slots: @@ -57,6 +61,7 @@ private slots: private: QString what; QString where; + QString md5; QDBusUnixFileDescriptor fd; QFileSystemWatcher watcher; }; diff --git a/helper/win/main.cpp b/helper/win/main.cpp index 9fc8c87..f857ef0 100644 --- a/helper/win/main.cpp +++ b/helper/win/main.cpp @@ -36,8 +36,8 @@ int main(int argc, char *argv[]) { if (app.arguments().count() == 3 && app.arguments()[1] == "restore") { new RestoreJob(app.arguments()[2]); - } else if (app.arguments().count() == 4 && app.arguments()[1] == "write") { - new WriteJob(app.arguments()[2], app.arguments()[3]); + } else if (app.arguments().count() == 5 && app.arguments()[1] == "write") { + new WriteJob(app.arguments()[2], app.arguments()[3], app.arguments()[4]); } else { QTextStream err(stderr); err << "Helper: Wrong arguments entered\n"; diff --git a/helper/win/win.pro b/helper/win/win.pro index afeceaf..206d991 100644 --- a/helper/win/win.pro +++ b/helper/win/win.pro @@ -4,7 +4,7 @@ include($$top_srcdir/deployment.pri) QT += core network -LIBS += -llzma +LIBS += -lisomd5 -llzma CONFIG += c++11 CONFIG += console diff --git a/helper/win/writejob.cpp b/helper/win/writejob.cpp index ff4382e..4c7e5ab 100644 --- a/helper/win/writejob.cpp +++ b/helper/win/writejob.cpp @@ -35,11 +35,15 @@ #include +#include "isomd5/libcheckisomd5.h" + const int BLOCK_SIZE = 512 * 128; -WriteJob::WriteJob(const QString &what, const QString &where) +WriteJob::WriteJob(const QString &what, const QString &where, const QString &md5_arg) : QObject(nullptr) -, what(what) { +, what(what) +, md5(md5_arg) +{ bool ok = false; this->where = where.toInt(&ok); @@ -50,6 +54,19 @@ WriteJob::WriteJob(const QString &what, const QString &where) QTimer::singleShot(0, this, &WriteJob::work); } +int WriteJob::staticOnMediaCheckAdvanced(void *data, long long offset, long long total) { + return ((WriteJob*)data)->onMediaCheckAdvanced(offset, total); +} + +int WriteJob::onMediaCheckAdvanced(long long offset, long long total) { + QTextStream out(stdout); + + Q_UNUSED(total); + out << offset << "\n"; + out.flush(); + return 0; +} + HANDLE WriteJob::openDrive(int physicalDriveNumber) { QTextStream err(stderr); @@ -240,9 +257,7 @@ void WriteJob::work() { } if (write_success) { - out.flush(); - err << "DONE\n"; - qApp->exit(0); + check(); } else { qApp->exit(4); } @@ -415,6 +430,9 @@ bool WriteJob::writePlain(HANDLE drive) { } void WriteJob::onFileChanged(const QString &path) { + QTextStream out(stdout); + QTextStream err(stderr); + const bool still_downloading = QFile::exists(path); if (still_downloading) { return; @@ -426,5 +444,78 @@ void WriteJob::onFileChanged(const QString &path) { return; } - work(); + // NOTE: let the app know that writing started + out << "WRITE\n"; + out.flush(); + + bool write_success = write(); + + // NOTE: try to write 2 times and sleep between + // attempts. Apparently needed on windows. + if (!write_success) { + out << "0\n"; + out.flush(); + QThread::sleep(5); + + write_success = write(); + } + + if (write_success) { + check(); + } else { + qApp->exit(4); + } +} + +bool WriteJob::check() { + QTextStream out(stdout); + QTextStream err(stdout); + + if (what.endsWith(".xz")) { + out << "NOT CHECKING BECAUSE IMAGE IS ZIPPED\n"; + out << "DONE\n"; + out.flush(); + err << "OK\n"; + err.flush(); + qApp->exit(0); + return true; + } + + if (what.endsWith(".xz")) { + out << "NOT CHECKING BECAUSE NO MD5 IS PROVIDED\n"; + out << "DONE\n"; + out.flush(); + err << "OK\n"; + err.flush(); + qApp->exit(0); + return true; + } + + out << "CHECK\n"; + out.flush(); + + HANDLE drive = openDrive(where); + + switch (mediaCheckFD(_open_osfhandle(reinterpret_cast(drive), 0), md5.toLocal8Bit().data(), &WriteJob::staticOnMediaCheckAdvanced, this)) { + case ISOMD5SUM_CHECK_NOT_FOUND: + case ISOMD5SUM_CHECK_PASSED: + out << "DONE\n"; + out.flush(); + err << "OK\n"; + err.flush(); + qApp->exit(0); + break; + case ISOMD5SUM_CHECK_FAILED: + err << tr("Your drive is probably damaged.") << "\n"; + err.flush(); + qApp->exit(1); + return false; + default: + err << tr("Unexpected error occurred during media check.") << "\n"; + err.flush(); + qApp->exit(1); + return false; + } + + return true; } diff --git a/helper/win/writejob.h b/helper/win/writejob.h index 6dc020a..52ebd0a 100644 --- a/helper/win/writejob.h +++ b/helper/win/writejob.h @@ -36,7 +36,10 @@ class WriteJob : public QObject { Q_OBJECT public: - explicit WriteJob(const QString &what, const QString &where); + explicit WriteJob(const QString &what, const QString &where, const QString &md5_arg); + + static int staticOnMediaCheckAdvanced(void *data, long long offset, long long total); + int onMediaCheckAdvanced(long long offset, long long total); private: HANDLE openDrive(int physicalDriveNumber); @@ -60,8 +63,11 @@ private slots: private: QString what; uint where; + QString md5; QFileSystemWatcher watcher; + + bool check(); }; #endif // WRITEJOB_H diff --git a/lib/isomd5/isomd5.pro b/lib/isomd5/isomd5.pro new file mode 100644 index 0000000..c84d09f --- /dev/null +++ b/lib/isomd5/isomd5.pro @@ -0,0 +1,13 @@ +TEMPLATE = lib + +CONFIG += staticlib + +QT += core + +DESTDIR = ../ + +HEADERS += libcheckisomd5.h + +SOURCES += libcheckisomd5.cpp + +QMAKE_MACOSX_DEPLOYMENT_TARGET = 10.9 diff --git a/lib/isomd5/libcheckisomd5.cpp b/lib/isomd5/libcheckisomd5.cpp new file mode 100644 index 0000000..cc18ccf --- /dev/null +++ b/lib/isomd5/libcheckisomd5.cpp @@ -0,0 +1,180 @@ +/* + * Copyright (C) 2001-2013 Red Hat, Inc. + * + * Michael Fulbright + * Dustin Kirkland + * Added support for checkpoint fragment sums; + * Exits media check as soon as bad fragment md5sum'ed + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +#define _LARGEFILE64_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "libcheckisomd5.h" + +#ifdef __APPLE__ +#define lseek64 lseek +#endif + +#ifdef _WIN32 +size_t getpagesize () { + return 2048; // not really necessary for Windows +} +#endif + +#define BUFSIZE 32768 +#define SIZE_OFFSET 84 + +#define MAX(x, y) ((x > y) ? x : y) +#define MIN(x, y) ((x < y) ? x : y) + +static int checkmd5sum(int fd, const char *mediasum, checkCallback cb, void *cbdata, long long size) { + // Md5 is empty, therefore md5 check not needed + if (mediasum[0] == '\0') { + return ISOMD5SUM_CHECK_PASSED; + } + + int pagesize = getpagesize(); + unsigned char *buf_unaligned = (unsigned char *) malloc((BUFSIZE + pagesize) * sizeof(unsigned char)); + unsigned char *buf = (buf_unaligned + (pagesize - ((uintptr_t) buf_unaligned % pagesize))); + + // Rewind + long long offset = lseek64(fd, 0LL, SEEK_SET); + + // Compute md5 + QCryptographicHash hash(QCryptographicHash::Md5); + + if (cb) { + cb(cbdata, 0, size); + } + + while (offset < size) { + ssize_t nattempt = MIN(size - offset, BUFSIZE); + + ssize_t nread = read(fd, buf, nattempt); + if (nread <= 0) + break; + + if (nread > nattempt) { + nread = nattempt; + lseek64(fd, offset + nread, SEEK_SET); + } + + hash.addData((const char *) buf, nread); + + offset = offset + nread; + if (cb && offset / nread % 256 == 0) { + if (cb(cbdata, offset, size)) { + free(buf_unaligned); + return ISOMD5SUM_CHECK_ABORTED; + } + } + } + + if (cb) { + cb(cbdata, size, size); + } + + free(buf_unaligned); + + const QByteArray computedsum_bytes = hash.result().toHex(); + const char *computed_sum = computedsum_bytes.constData(); + + const bool sums_match = (memcmp(computed_sum, mediasum, computedsum_bytes.size()) == 0); + + if (sums_match) { + return ISOMD5SUM_CHECK_PASSED; + } else { + return ISOMD5SUM_CHECK_FAILED; + } +} + +int mediaCheckFile(const char *file, const char *md5,checkCallback cb, void *cbdata) { + int fd; + +#ifdef _WIN32 + fd = open(file, O_RDONLY | O_BINARY); +#else + fd = open(file, O_RDONLY); +#endif + + if (fd < 0) { + return ISOMD5SUM_FILE_NOT_FOUND; + } + + // Calculate file size + long long size = lseek64(fd, 0L, SEEK_END); + + int rc = checkmd5sum(fd, md5, cb, cbdata, size); + + close(fd); + + return rc; +} + +int mediaCheckFD(int fd, const char *md5, checkCallback cb, void *cbdata) { + if (fd < 0) { + return ISOMD5SUM_FILE_NOT_FOUND; + } + + // NOTE: files that are FD(written to drive) are implicitly always iso's + // Get size + int pagesize = getpagesize(); + unsigned char *buf_unaligned = (unsigned char *) malloc((BUFSIZE + pagesize) * sizeof(unsigned char)); + unsigned char *buf = (buf_unaligned + (pagesize - ((uintptr_t) buf_unaligned % pagesize))); + if (lseek64(fd, (16LL * 2048LL), SEEK_SET) == -1) { + free(buf_unaligned); + return ISOMD5SUM_CHECK_NOT_FOUND; + } + + long long offset = (16LL * 2048LL); + for (;1;) { + if (read(fd, buf, 2048) <= 0) { + free(buf_unaligned); + return ISOMD5SUM_CHECK_NOT_FOUND; + } + + if (buf[0] == 1) { + /* found primary volume descriptor */ + break; + } else if (buf[0] == 255) { + /* hit end and didn't find primary volume descriptor */ + free(buf_unaligned); + return ISOMD5SUM_CHECK_NOT_FOUND; + } + offset += 2048LL; + } + + // Get size from pvd + long long size = (buf[SIZE_OFFSET] * 0x1000000 + buf[SIZE_OFFSET + 1] * 0x10000 + buf[SIZE_OFFSET + 2] * 0x100 + buf[SIZE_OFFSET + 3]) * 2048LL; + + free(buf_unaligned); + + int rc = checkmd5sum(fd, md5, cb, cbdata, size); + + return rc; +} diff --git a/lib/isomd5/libcheckisomd5.h b/lib/isomd5/libcheckisomd5.h new file mode 100644 index 0000000..6da2015 --- /dev/null +++ b/lib/isomd5/libcheckisomd5.h @@ -0,0 +1,18 @@ +#ifndef __LIBCHECKISOMD5_H__ +#define __LIBCHECKISOMD5_H__ + +#define ISOMD5SUM_CHECK_PASSED 1 +#define ISOMD5SUM_CHECK_FAILED 0 +#define ISOMD5SUM_CHECK_ABORTED 2 +#define ISOMD5SUM_CHECK_NOT_FOUND -1 +#define ISOMD5SUM_FILE_NOT_FOUND -2 + +/* for non-zero return value, check is aborted */ +typedef int (*checkCallback)(void *, long long offset, long long total); + +int mediaCheckFile(const char *iso, const char *md5, checkCallback cb, void *cbdata); +int mediaCheckFD(int fd, const char *md5, checkCallback cb, void *cbdata); +int printMD5SUM(char *file); + +#endif + diff --git a/lib/lib.pro b/lib/lib.pro new file mode 100644 index 0000000..6160dd5 --- /dev/null +++ b/lib/lib.pro @@ -0,0 +1,3 @@ +TEMPLATE = subdirs + +SUBDIRS = isomd5 diff --git a/mediawriter.pro b/mediawriter.pro index 1622b82..cb225fb 100644 --- a/mediawriter.pro +++ b/mediawriter.pro @@ -1,3 +1,6 @@ TEMPLATE = subdirs -SUBDIRS = app helper +SUBDIRS = lib app helper + +app.depends = lib +helper.depends = lib