Add z-check-tools-sha

include/vcpkg/base/contractual-constants.h

@@ -218,6 +218,7 @@ namespace vcpkg
     inline constexpr StringLiteral SwitchFailingAbiLog = "failing-abi-log";
     inline constexpr StringLiteral SwitchFailureLogs = "failure-logs";
     inline constexpr StringLiteral SwitchFeatureFlags = "feature-flags";
+    inline constexpr StringLiteral SwitchFix = "fix";
     inline constexpr StringLiteral SwitchForce = "force";
     inline constexpr StringLiteral SwitchFormat = "format";
     inline constexpr StringLiteral SwitchForMergeWith = "for-merge-with";
@@ -253,6 +254,7 @@ namespace vcpkg
     inline constexpr StringLiteral SwitchNuGetVersion = "nuget-version";
     inline constexpr StringLiteral SwitchOnlyBinarycaching = "only-binarycaching";
     inline constexpr StringLiteral SwitchOnlyDownloads = "only-downloads";
+    inline constexpr StringLiteral SwitchOnlyWithName = "only-with-name";
     inline constexpr StringLiteral SwitchOsx = "osx";
     inline constexpr StringLiteral SwitchOutdated = "outdated";
     inline constexpr StringLiteral SwitchOutput = "output";

include/vcpkg/base/message-data.inc.h

@@ -131,6 +131,7 @@ DECLARE_MESSAGE(AllFormatArgsUnbalancedBraces,
                 "example of {value} is 'foo bar {'",
                 "unbalanced brace in format string \"{value}\"")
 DECLARE_MESSAGE(AllPackagesAreUpdated, (), "", "All installed packages are up-to-date.")
+DECLARE_MESSAGE(AllShasValid, (), "sha = sha512 of url", "All checked sha's are valid.")
 DECLARE_MESSAGE(AlreadyInstalled, (msg::spec), "", "{spec} is already installed")
 DECLARE_MESSAGE(AlreadyInstalledNotHead,
                 (msg::spec),
@@ -595,6 +596,12 @@ DECLARE_MESSAGE(CmdCheckSupportExample1,
                 "This is a command line, only the <>s part should be localized",
                 "vcpkg x-check-support <port name>")
 DECLARE_MESSAGE(CmdCheckSupportSynopsis, (), "", "Tests whether a port is supported without building it")
+DECLARE_MESSAGE(CmdCheckToolsShaSynopsis,
+                (),
+                "",
+                "Checks the sha512 entries in a tools data file by downloading all entries and computing the hashes")
+DECLARE_MESSAGE(CmdCheckToolsShaSwitchFix, (), "", "Fixes the sha entry in the given file")
+DECLARE_MESSAGE(CmdCheckToolsShaSwitchOnlyWithName, (), "", "Only check entries with the given name")
 DECLARE_MESSAGE(CmdCreateExample1,
                 (),
                 "This is a command line, only the <>s part should be localized",
@@ -1095,6 +1102,7 @@ DECLARE_MESSAGE(DownloadingFileFirstAuthoritativeSource, (msg::path, msg::url),
 DECLARE_MESSAGE(DownloadingUrlToFile, (msg::url, msg::path), "", "Downloading {url} -> {path}")
 DECLARE_MESSAGE(DownloadingVcpkgStandaloneBundle, (msg::version), "", "Downloading standalone bundle {version}.")
 DECLARE_MESSAGE(DownloadingVcpkgStandaloneBundleLatest, (), "", "Downloading latest standalone bundle.")
+DECLARE_MESSAGE(DownloadingTools, (msg::count), "", "Downloading {count} tools")
 DECLARE_MESSAGE(DownloadOrUrl, (msg::url), "", "or {url}")
 DECLARE_MESSAGE(DownloadTryingAuthoritativeSource, (msg::url), "", "Trying {url}")
 DECLARE_MESSAGE(DownloadRootsDir, (msg::env_var), "", "Downloads directory (default: {env_var})")
@@ -1396,6 +1404,7 @@ DECLARE_MESSAGE(FindCommandFirstArg,
                 "'find', 'artifact', and 'port' are vcpkg specific terms and should not be translated.",
                 "The first argument to 'find' must be 'artifact' or 'port' .")
 DECLARE_MESSAGE(FishCompletion, (msg::path), "", "vcpkg fish completion is already added at \"{path}\".")
+DECLARE_MESSAGE(FixedEntriesInFile, (msg::count, msg::path), "", "Fixed {count} entries in {path}.")
 DECLARE_MESSAGE(FloatingPointConstTooBig, (msg::count), "", "Floating point constant too big: {count}")
 DECLARE_MESSAGE(FollowingPackagesMissingControl,
                 (),
@@ -2197,6 +2206,10 @@ DECLARE_MESSAGE(NoCoreFeatureAllowedInNonFailBaselineEntry,
                 (msg::value),
                 "{value} is a keyword",
                 "'core' is not allowed in the list of features if the entry is of type '{value}'")
+DECLARE_MESSAGE(NoEntryWithName,
+                (msg::value),
+                "{value} is the name of an entry",
+                "No entry found with name '{value}' and a url.")
 DECLARE_MESSAGE(NoError, (), "", "no error")
 DECLARE_MESSAGE(NoInstalledPackages,
                 (),

include/vcpkg/commands.z-check-tools-sha.h

@@ -0,0 +1,11 @@
+#pragma once
+
+#include <vcpkg/base/fwd/files.h>
+
+#include <vcpkg/fwd/vcpkgcmdarguments.h>
+
+namespace vcpkg
+{
+    extern const CommandMetadata CommandCheckToolsShaMetadata;
+    void command_check_tools_sha_and_exit(const VcpkgCmdArguments& args, const Filesystem& paths);
+}

locales/messages.json

@@ -115,6 +115,8 @@
   "AllFormatArgsUnbalancedBraces": "unbalanced brace in format string \"{value}\"",
   "_AllFormatArgsUnbalancedBraces.comment": "example of {value} is 'foo bar {'",
   "AllPackagesAreUpdated": "All installed packages are up-to-date.",
+  "AllShasValid": "All checked sha's are valid.",
+  "_AllShasValid.comment": "sha = sha512 of url",
   "AlreadyInstalled": "{spec} is already installed",
   "_AlreadyInstalled.comment": "An example of {spec} is zlib:x64-windows.",
   "AlreadyInstalledNotHead": "{spec} is already installed -- not building from HEAD",
@@ -343,6 +345,9 @@
   "CmdCheckSupportExample1": "vcpkg x-check-support <port name>",
   "_CmdCheckSupportExample1.comment": "This is a command line, only the <>s part should be localized",
   "CmdCheckSupportSynopsis": "Tests whether a port is supported without building it",
+  "CmdCheckToolsShaSwitchFix": "Fixes the sha entry in the given file",
+  "CmdCheckToolsShaSwitchOnlyWithName": "Only check entries with the given name",
+  "CmdCheckToolsShaSynopsis": "Checks the sha512 entries in a tools data file by downloading all entries and computing the hashes",
   "CmdCiCleanSynopsis": "Clears all files to prepare for a CI run",
   "_CmdCiCleanSynopsis.comment": "CI is continuous integration (building everything together)",
   "CmdCiSynopsis": "Tries building all ports for CI testing",
@@ -639,6 +644,8 @@
   "_DownloadingFileFirstAuthoritativeSource.comment": "An example of {path} is /foo/bar. An example of {url} is https://github.com/microsoft/vcpkg.",
   "DownloadingPortableToolVersionX": "A suitable version of {tool_name} was not found (required v{version}).",
   "_DownloadingPortableToolVersionX.comment": "An example of {tool_name} is signtool. An example of {version} is 1.3.8.",
+  "DownloadingTools": "Downloading {count} tools",
+  "_DownloadingTools.comment": "An example of {count} is 42.",
   "DownloadingUrlToFile": "Downloading {url} -> {path}",
   "_DownloadingUrlToFile.comment": "An example of {url} is https://github.com/microsoft/vcpkg. An example of {path} is /foo/bar.",
   "DownloadingVcpkgStandaloneBundle": "Downloading standalone bundle {version}.",
@@ -837,6 +844,8 @@
   "_FindVersionArtifactsOnly.comment": "'--version', 'vcpkg search', and 'vcpkg find port' are command lines that must not be localized",
   "FishCompletion": "vcpkg fish completion is already added at \"{path}\".",
   "_FishCompletion.comment": "An example of {path} is /foo/bar.",
+  "FixedEntriesInFile": "Fixed {count} entries in {path}.",
+  "_FixedEntriesInFile.comment": "An example of {count} is 42. An example of {path} is /foo/bar.",
   "FloatingPointConstTooBig": "Floating point constant too big: {count}",
   "_FloatingPointConstTooBig.comment": "An example of {count} is 42.",
   "FollowingPackagesMissingControl": "The following packages do not have a valid CONTROL or vcpkg.json:",
@@ -1208,6 +1217,8 @@
   "NewVersionCannotBeEmpty": "--version cannot be empty.",
   "NoCoreFeatureAllowedInNonFailBaselineEntry": "'core' is not allowed in the list of features if the entry is of type '{value}'",
   "_NoCoreFeatureAllowedInNonFailBaselineEntry.comment": "{value} is a keyword",
+  "NoEntryWithName": "No entry found with name '{value}' and a url.",
+  "_NoEntryWithName.comment": "{value} is the name of an entry",
   "NoError": "no error",
   "NoInstalledPackages": "No packages are installed. Did you mean `search`?",
   "_NoInstalledPackages.comment": "The name 'search' is the name of a command that is not localized.",

src/vcpkg/commands.cpp

@@ -50,6 +50,7 @@
 #include <vcpkg/commands.z-applocal.h>
 #include <vcpkg/commands.z-ce.h>
 #include <vcpkg/commands.z-changelog.h>
+#include <vcpkg/commands.z-check-tools-sha.h>
 #include <vcpkg/commands.z-extract.h>
 #include <vcpkg/commands.z-generate-message-map.h>
 #include <vcpkg/commands.z-preregister-telemetry.h>
@@ -65,6 +66,7 @@ namespace vcpkg
         {CommandDownloadMetadata, command_download_and_exit},
         {CommandFormatFeatureBaselineMetadata, command_format_feature_baseline_and_exit},
         {CommandHashMetadata, command_hash_and_exit},
+        {CommandCheckToolsShaMetadata, command_check_tools_sha_and_exit},
         {CommandInitRegistryMetadata, command_init_registry_and_exit},
         {CommandVersionMetadata, command_version_and_exit},
 #if defined(_WIN32)

src/vcpkg/commands.z-check-tools-sha.cpp

@@ -0,0 +1,139 @@
+#include <vcpkg/base/contractual-constants.h>
+#include <vcpkg/base/downloads.h>
+#include <vcpkg/base/files.h>
+#include <vcpkg/base/hash.h>
+#include <vcpkg/base/util.h>
+
+#include <vcpkg/commands.z-check-tools-sha.h>
+#include <vcpkg/tools.test.h>
+#include <vcpkg/vcpkgcmdarguments.h>
+
+namespace vcpkg
+{
+
+    static constexpr CommandSwitch CHECK_TOOL_SWITCHES[] = {
+        {SwitchFix, msgCmdCheckToolsShaSwitchFix},
+    };
+
+    static constexpr CommandSetting CHECK_TOOL_SETTINGS[] = {
+        {SwitchOnlyWithName, msgCmdCheckToolsShaSwitchOnlyWithName},
+    };
+
+    constexpr CommandMetadata CommandCheckToolsShaMetadata{
+        "z-check-tools-sha",
+        msgCmdCheckToolsShaSynopsis,
+        {"vcpkg z-check-tools-sha scripts/vcpkg-tools.json"},
+        Undocumented,
+        AutocompletePriority::Internal,
+        1,
+        1,
+        {CHECK_TOOL_SWITCHES, CHECK_TOOL_SETTINGS},
+        nullptr,
+    };
+
+    void command_check_tools_sha_and_exit(const VcpkgCmdArguments& args, const Filesystem& fs)
+    {
+        const auto parsed = args.parse_arguments(CommandCheckToolsShaMetadata);
+
+        const auto file_to_check = (fs.current_path(VCPKG_LINE_INFO) / parsed.command_arguments[0]).lexically_normal();
+
+        const auto dest_path = fs.create_or_get_temp_directory(VCPKG_LINE_INFO);
+
+        auto content = fs.read_contents(file_to_check, VCPKG_LINE_INFO);
+
+        auto data = parse_tool_data(content, file_to_check).value_or_exit(VCPKG_LINE_INFO);
+
+        auto only_name_iter = parsed.settings.find(SwitchOnlyWithName);
+
+        std::unordered_map<std::string, std::string> url_to_sha;
+
+        std::vector<std::pair<std::string, Path>> urlAndPaths;
+        for (auto& entry : data)
+        {
+            if (entry.url.empty()) continue;
+            if (only_name_iter != parsed.settings.end())
+            {
+                if (entry.tool != only_name_iter->second)
+                {
+                    continue;
+                }
+            }
+            auto iter = url_to_sha.emplace(entry.url, entry.sha512);
+
+            Checks::check_exit(VCPKG_LINE_INFO, iter.first->second == entry.sha512);
+            if (iter.second)
+            {
+                urlAndPaths.emplace_back(entry.url, dest_path / entry.archiveName + " - " + entry.sha512.substr(0, 10));
+            }
+        }
+        if (urlAndPaths.empty())
+        {
+            Checks::msg_exit_with_message(
+                VCPKG_LINE_INFO,
+                msg::format(msgNoEntryWithName,
+                            msg::value = (only_name_iter != parsed.settings.end() ? only_name_iter->second : "*")));
+        }
+
+        msg::println(msgDownloadingTools, msg::count = urlAndPaths.size());
+        auto result = download_files_no_cache(console_diagnostic_context, urlAndPaths, {}, {});
+
+        std::unordered_map<std::string, std::string> url_to_fixed_sha;
+        auto http_codes_iter = result.begin();
+        bool has_http_error = false;
+        bool has_sha_error = false;
+        for (auto& urlAndPath : urlAndPaths)
+        {
+            if (*http_codes_iter == 200)
+            {
+                auto sha =
+                    Hash::get_file_hash(fs, urlAndPath.second, Hash::Algorithm::Sha512).value_or_exit(VCPKG_LINE_INFO);
+                if (url_to_sha[urlAndPath.first] != sha)
+                {
+                    msg::println(msgDownloadFailedHashMismatch, msg::url = urlAndPath.first);
+                    msg::println(msgDownloadFailedHashMismatchExpectedHash, msg::sha = url_to_sha[urlAndPath.first]);
+                    msg::println(msgDownloadFailedHashMismatchActualHash, msg::sha = sha);
+                    msg::println();
+                    has_sha_error = true;
+                    url_to_fixed_sha[urlAndPath.first] = sha;
+                }
+            }
+            else
+            {
+                msg::println(msgDownloadFailedStatusCode, msg::url = urlAndPath.first, msg::value = *http_codes_iter);
+                has_http_error = true;
+            }
+            fs.remove(urlAndPath.second, VCPKG_LINE_INFO);
+            ++http_codes_iter;
+        }
+
+        if (!has_sha_error)
+        {
+            msg::println(msgAllShasValid);
+        }
+
+        if (!url_to_fixed_sha.empty() && Util::Sets::contains(parsed.switches, SwitchFix))
+        {
+            Json::Object as_object = Json::parse_object(content, file_to_check).value_or_exit(VCPKG_LINE_INFO);
+            int fixed = 0;
+            for (auto&& entry : as_object["tools"].array(VCPKG_LINE_INFO))
+            {
+                auto maybe_url = entry.object(VCPKG_LINE_INFO).get("url");
+                if (maybe_url)
+                {
+                    auto iter = url_to_fixed_sha.find(maybe_url->string(VCPKG_LINE_INFO).to_string());
+                    if (iter != url_to_fixed_sha.end())
+                    {
+                        entry.object(VCPKG_LINE_INFO).insert_or_replace("sha512", iter->second);
+                        ++fixed;
+                    }
+                }
+            }
+
+            fs.write_contents(file_to_check, Json::stringify(as_object), VCPKG_LINE_INFO);
+            msg::println(msgFixedEntriesInFile, msg::count = fixed, msg::path = file_to_check);
+            has_sha_error = false;
+        }
+
+        Checks::exit_with_code(VCPKG_LINE_INFO, (has_sha_error || has_http_error) ? EXIT_FAILURE : EXIT_SUCCESS);
+    }
+}