Add an S3-compatible remote cache storage plugin

Directory.Packages.props

@@ -8,6 +8,7 @@
     <RestoreUseStaticGraphEvaluation>true</RestoreUseStaticGraphEvaluation>
   </PropertyGroup>
   <ItemGroup>
+    <PackageVersion Include="AWSSDK.S3" Version="3.7.416.12" />
     <PackageVersion Include="CopyOnWrite" Version="0.5.0" />
     <PackageVersion Include="DotNet.Glob" Version="2.0.3" />
     <PackageVersion Include="ILRepack" Version="2.0.44" />

MSBuildCache.sln

@@ -32,10 +32,14 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.MSBuildCache.Azur
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.MSBuildCache.Local", "src\Local\Microsoft.MSBuildCache.Local.csproj", "{F6586428-E047-42C8-B0AC-048DF6DFAF18}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.MSBuildCache.S3", "src\S3\Microsoft.MSBuildCache.S3.csproj", "{9AF00174-D361-4143-8233-5A1A52965261}"
+EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.MSBuildCache.Repack.Tests", "src\Repack.Tests\Microsoft.MSBuildCache.Repack.Tests.csproj", "{3BCB6452-B087-4A03-8418-C79F2715DDE7}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.MSBuildCache.AzurePipelines.Tests", "src\AzurePipelines.Tests\Microsoft.MSBuildCache.AzurePipelines.Tests.csproj", "{61A86AEA-F043-4CC4-B60B-A040C5C36194}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.MSBuildCache.S3.Tests", "src\S3.Tests\Microsoft.MSBuildCache.S3.Tests.csproj", "{1D0A6E44-4B1F-4CE2-9E3B-7F5C2A8D6B10}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -66,6 +70,10 @@ Global
 		{F6586428-E047-42C8-B0AC-048DF6DFAF18}.Debug|x64.Build.0 = Debug|x64
 		{F6586428-E047-42C8-B0AC-048DF6DFAF18}.Release|x64.ActiveCfg = Release|x64
 		{F6586428-E047-42C8-B0AC-048DF6DFAF18}.Release|x64.Build.0 = Release|x64
+		{9AF00174-D361-4143-8233-5A1A52965261}.Debug|x64.ActiveCfg = Debug|x64
+		{9AF00174-D361-4143-8233-5A1A52965261}.Debug|x64.Build.0 = Debug|x64
+		{9AF00174-D361-4143-8233-5A1A52965261}.Release|x64.ActiveCfg = Release|x64
+		{9AF00174-D361-4143-8233-5A1A52965261}.Release|x64.Build.0 = Release|x64
 		{3BCB6452-B087-4A03-8418-C79F2715DDE7}.Debug|x64.ActiveCfg = Debug|x64
 		{3BCB6452-B087-4A03-8418-C79F2715DDE7}.Debug|x64.Build.0 = Debug|x64
 		{3BCB6452-B087-4A03-8418-C79F2715DDE7}.Release|x64.ActiveCfg = Release|x64
@@ -74,6 +82,10 @@ Global
 		{61A86AEA-F043-4CC4-B60B-A040C5C36194}.Debug|x64.Build.0 = Debug|x64
 		{61A86AEA-F043-4CC4-B60B-A040C5C36194}.Release|x64.ActiveCfg = Release|x64
 		{61A86AEA-F043-4CC4-B60B-A040C5C36194}.Release|x64.Build.0 = Release|x64
+		{1D0A6E44-4B1F-4CE2-9E3B-7F5C2A8D6B10}.Debug|x64.ActiveCfg = Debug|x64
+		{1D0A6E44-4B1F-4CE2-9E3B-7F5C2A8D6B10}.Debug|x64.Build.0 = Debug|x64
+		{1D0A6E44-4B1F-4CE2-9E3B-7F5C2A8D6B10}.Release|x64.ActiveCfg = Release|x64
+		{1D0A6E44-4B1F-4CE2-9E3B-7F5C2A8D6B10}.Release|x64.Build.0 = Release|x64
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -85,8 +97,10 @@ Global
 		{D0195D37-E001-4283-B51A-A0B51B1D54D1} = {EFFB5949-347C-4F28-8964-571D5C6B6209}
 		{97357681-C75E-445D-8547-46F312D01CED} = {EFFB5949-347C-4F28-8964-571D5C6B6209}
 		{F6586428-E047-42C8-B0AC-048DF6DFAF18} = {EFFB5949-347C-4F28-8964-571D5C6B6209}
+		{9AF00174-D361-4143-8233-5A1A52965261} = {EFFB5949-347C-4F28-8964-571D5C6B6209}
 		{3BCB6452-B087-4A03-8418-C79F2715DDE7} = {EFFB5949-347C-4F28-8964-571D5C6B6209}
 		{61A86AEA-F043-4CC4-B60B-A040C5C36194} = {EFFB5949-347C-4F28-8964-571D5C6B6209}
+		{1D0A6E44-4B1F-4CE2-9E3B-7F5C2A8D6B10} = {EFFB5949-347C-4F28-8964-571D5C6B6209}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {F1CDA78F-A666-431B-BF44-56DA7DF193BA}

README.md

@@ -167,6 +167,48 @@ In the cases where an Azure credential is acquired, the following methods will b
 3. If `$(MSBuildCacheManagedIdentityClientId)` is set, it will be used as a user-assigned [managed identity](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/overview). This is recommended for non-interactive scenarios.
 4. If `$(MSBuildCacheAllowInteractiveAuth)` is true, credentials will be obtained interactively. This is recommended for developer scenarios.
 
+### Microsoft.MSBuildCache.S3
+[![NuGet Version](https://img.shields.io/nuget/v/Microsoft.MSBuildCache.S3.svg)](https://www.nuget.org/packages/Microsoft.MSBuildCache.S3)
+[![NuGet Downloads](https://img.shields.io/nuget/dt/Microsoft.MSBuildCache.S3.svg)](https://www.nuget.org/packages/Microsoft.MSBuildCache.S3)
+
+This implementation uses [Amazon S3](https://aws.amazon.com/s3/), or an S3-compatible store such as [MinIO](https://min.io/), as the cache storage.
+
+> [!WARNING]
+> This implementation does not yet have a robust security model. All builds using this will need write access to the bucket, so for example an external contributor could send a PR which would write/overwrite arbitrary content which could then be used by CI builds. Builds using this plugin must be restricted to trusted team members. Use at your own risk.
+
+These settings are available in addition to the [Common Settings](#common-settings):
+
+| MSBuild Property Name | Setting Type | Default value | Description |
+| ------------- | ------------ | ------------- | ----------- |
+| `$(MSBuildCacheS3BucketName)` | `string` | | The bucket used for cache storage. Required. |
+| `$(MSBuildCacheS3Region)` | `string` | "us-east-1" | The AWS region. Also used as the signing region when `$(MSBuildCacheS3ServiceUrl)` is set. |
+| `$(MSBuildCacheS3ServiceUrl)` | `Uri` | | The service url of an S3-compatible store, for example `http://localhost:9000` for MinIO. When unset, the AWS endpoint for the region is used. |
+| `$(MSBuildCacheS3ForcePathStyle)` | `bool` | true when `$(MSBuildCacheS3ServiceUrl)` is set, otherwise false | Whether to use path-style addressing. Most S3-compatible stores require this. |
+| `$(MSBuildCacheS3KeyPrefix)` | `string` | "msbuildcache" | The key prefix under which all cache objects are stored. |
+| `$(MSBuildCacheS3MultipartThresholdBytes)` | `long` | 33554432 (32 MB) | Objects at or above this size are transferred as multiple parts in parallel instead of as a single request. |
+| `$(MSBuildCacheS3MultipartPartSizeBytes)` | `long` | 8388608 (8 MB) | The part size for multipart transfers. Values below S3's 5 MB minimum are raised to it. |
+| `$(MSBuildCacheS3MaxConcurrentPartsPerObject)` | `int` | 8 | How many parts of a single object are transferred concurrently. Ranged requests across all multipart transfers are additionally bounded by `$(MSBuildCacheMaxConcurrentCacheContentOperations)`. |
+
+The bucket is expected to already exist. Credentials are deliberately not settings, since settings are configured through item metadata and so end up in MSBuild logs. They are resolved as follows, in priority order:
+1. If [`AWSCredentials`](https://docs.aws.amazon.com/sdkfornet/v3/apidocs/items/Runtime/TAWSCredentials.html) are provided directly in the plugin's constructor, they are used. This only applies when using the programmatic project cache API.
+2. Otherwise the [default AWS credential chain](https://docs.aws.amazon.com/sdk-for-net/v3/developer-guide/creds-assign.html) is used, which covers the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables, the shared credentials file, and instance and task roles.
+
+For example, to use a local MinIO instance:
+
+```xml
+<PropertyGroup>
+  <MSBuildCachePackage>Microsoft.MSBuildCache.S3</MSBuildCachePackage>
+  <MSBuildCacheS3BucketName>msbuildcache</MSBuildCacheS3BucketName>
+  <MSBuildCacheS3ServiceUrl>http://localhost:9000</MSBuildCacheS3ServiceUrl>
+</PropertyGroup>
+```
+
+The end-to-end test for this plugin starts a local [moto](https://github.com/getmoto/moto) server, so it needs no AWS account and no container runtime, only [uv](https://docs.astral.sh/uv) or an installed `moto_server`:
+
+```
+.\tests\s3.ps1
+```
+
 ## Other Packages
 
 ### Microsoft.MSBuildCache.SharedCompilation

src/Common/HexUtilities.cs

@@ -2,6 +2,10 @@
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
 using System;
+#if NETFRAMEWORK
+using System.Globalization;
+using System.Text;
+#endif
 
 namespace Microsoft.MSBuildCache;
 
@@ -23,6 +27,24 @@ public static byte[] HexToBytes(string? hex)
             ? Array.Empty<byte>()
             : HexToBytes(hex.AsSpan());
 
+    /// <summary>
+    /// Formats bytes as an uppercase hexadecimal string without a '0x' prefix, ie the inverse of <see cref="HexToBytes(string?)"/>.
+    /// </summary>
+    public static string BytesToHex(ReadOnlySpan<byte> bytes)
+    {
+#if NETFRAMEWORK
+        StringBuilder result = new(bytes.Length * 2);
+        foreach (byte b in bytes)
+        {
+            result.Append(b.ToString("X2", CultureInfo.InvariantCulture));
+        }
+
+        return result.ToString();
+#else
+        return Convert.ToHexString(bytes);
+#endif
+    }
+
     /// <summary>
     /// Parses hexadecimal strings the form '1234abcd' or '0x9876fedb' into
     /// an array of bytes.

src/S3.Tests/Microsoft.MSBuildCache.S3.Tests.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="MSTest.Sdk">
+  <PropertyGroup>
+    <!-- Only supports x64 due to the RocksDB dependency -->
+    <Platform>x64</Platform>
+    <Platforms>$(Platform)</Platforms>
+    <TargetFramework>net9.0</TargetFramework>
+    <RootNamespace>Microsoft.MSBuildCache.S3.Tests</RootNamespace>
+    <!-- MSTest requires public test classes even though the test host is an application. -->
+    <NoWarn>$(NoWarn);CA1515</NoWarn>
+  </PropertyGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\S3\Microsoft.MSBuildCache.S3.csproj" />
+  </ItemGroup>
+</Project>

src/S3.Tests/S3ObjectKeysTests.cs

@@ -0,0 +1,154 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using BuildXL.Cache.ContentStore.Hashing;
+using BuildXL.Cache.MemoizationStore.Interfaces.Sessions;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace Microsoft.MSBuildCache.S3.Tests;
+
+[TestClass]
+public class S3ObjectKeysTests
+{
+    private const string KeyPrefix = "msbuildcache";
+    private const string Universe = "abc123";
+
+    private static readonly Fingerprint WeakFingerprint = new(new byte[] { 1, 2, 3, 4 });
+    private static readonly ContentHash PathSetHash = new(HashType.Vso0, new byte[33]);
+
+    private static S3ObjectKeys CreateKeys(string keyPrefix = KeyPrefix, HashType hashType = HashType.Vso0, string universe = Universe)
+        => new(keyPrefix, hashType, universe);
+
+    private static StrongFingerprint CreateStrongFingerprint(byte[] strongFingerprintBytes)
+        => new(WeakFingerprint, new Selector(PathSetHash, strongFingerprintBytes));
+
+    [TestMethod]
+    public void KeysAreScopedByPrefixAndUniverse()
+    {
+        S3ObjectKeys keys = CreateKeys();
+
+        string casKey = keys.GetCas(PathSetHash);
+
+        StringAssert.StartsWith(casKey, $"{KeyPrefix}/", StringComparison.Ordinal);
+        StringAssert.Contains(casKey, $"/{Universe}/", StringComparison.Ordinal);
+        StringAssert.Contains(casKey, "/cas/", StringComparison.Ordinal);
+    }
+
+    [TestMethod]
+    public void KeysAreScopedByHashType()
+    {
+        string vso0 = CreateKeys(hashType: HashType.Vso0).GetCas(PathSetHash);
+        string sha256 = CreateKeys(hashType: HashType.SHA256).GetCas(PathSetHash);
+
+        Assert.AreNotEqual(vso0, sha256);
+    }
+
+    [TestMethod]
+    public void KeysAreScopedByUniverse()
+    {
+        string one = CreateKeys(universe: "one").GetCas(PathSetHash);
+        string two = CreateKeys(universe: "two").GetCas(PathSetHash);
+
+        Assert.AreNotEqual(one, two);
+    }
+
+    [TestMethod]
+    [DataRow("")]
+    [DataRow("   ")]
+    [DataRow("/")]
+    public void EmptyKeyPrefixFallsBackToDefault(string keyPrefix)
+    {
+        string casKey = CreateKeys(keyPrefix: keyPrefix).GetCas(PathSetHash);
+
+        StringAssert.StartsWith(casKey, "msbuildcache/", StringComparison.Ordinal);
+    }
+
+    [TestMethod]
+    [DataRow("prefix/")]
+    [DataRow("/prefix")]
+    [DataRow(" prefix ")]
+    public void KeyPrefixIsNormalized(string keyPrefix)
+    {
+        string casKey = CreateKeys(keyPrefix: keyPrefix).GetCas(PathSetHash);
+
+        StringAssert.StartsWith(casKey, "prefix/", StringComparison.Ordinal);
+        Assert.IsFalse(casKey.Contains("//", StringComparison.Ordinal), $"Key '{casKey}' contains an empty segment.");
+    }
+
+    [TestMethod]
+    public void SerializedHashesDoNotAppearInKeys()
+    {
+        // ':' is legal in an S3 key but breaks tooling which maps keys onto file paths.
+        string casKey = CreateKeys().GetCas(PathSetHash);
+
+        Assert.IsFalse(casKey.Contains(':', StringComparison.Ordinal), $"Key '{casKey}' contains a colon.");
+    }
+
+    [TestMethod]
+    public void EntryAndSelectorKeysDiffer()
+    {
+        S3ObjectKeys keys = CreateKeys();
+        StrongFingerprint fingerprint = CreateStrongFingerprint(new byte[] { 42 });
+
+        Assert.AreNotEqual(keys.GetEntry(fingerprint), keys.GetSelector(fingerprint));
+    }
+
+    [TestMethod]
+    public void SelectorPrefixMatchesItsSelectors()
+    {
+        S3ObjectKeys keys = CreateKeys();
+        StrongFingerprint fingerprint = CreateStrongFingerprint(new byte[] { 42 });
+
+        StringAssert.StartsWith(keys.GetSelector(fingerprint), keys.GetSelectorPrefix(WeakFingerprint), StringComparison.Ordinal);
+    }
+
+    [TestMethod]
+    // A single zero byte is what the base cache client uses for its empty selector.
+    [DataRow(new byte[] { 0 })]
+    [DataRow(new byte[] { 0xDE, 0xAD, 0xBE, 0xEF })]
+    [DataRow(new byte[0])]
+    public void SelectorsRoundTrip(byte[] strongFingerprintBytes)
+    {
+        S3ObjectKeys keys = CreateKeys();
+        StrongFingerprint fingerprint = CreateStrongFingerprint(strongFingerprintBytes);
+
+        bool parsed = keys.TryParseSelector(keys.GetSelector(fingerprint), WeakFingerprint, out Selector selector);
+
+        Assert.IsTrue(parsed);
+        Assert.AreEqual(fingerprint.Selector, selector);
+    }
+
+    [TestMethod]
+    public void SelectorsOfAnotherWeakFingerprintAreRejected()
+    {
+        S3ObjectKeys keys = CreateKeys();
+        string selectorKey = keys.GetSelector(CreateStrongFingerprint(new byte[] { 42 }));
+
+        Assert.IsFalse(keys.TryParseSelector(selectorKey, new Fingerprint(new byte[] { 9, 9, 9, 9 }), out _));
+    }
+
+    [TestMethod]
+    public void SelectorsOfAnotherUniverseAreRejected()
+    {
+        string selectorKey = CreateKeys(universe: "other").GetSelector(CreateStrongFingerprint(new byte[] { 42 }));
+
+        Assert.IsFalse(CreateKeys().TryParseSelector(selectorKey, WeakFingerprint, out _));
+    }
+
+    [TestMethod]
+    public void MalformedSelectorsAreRejected()
+    {
+        S3ObjectKeys keys = CreateKeys();
+        string prefix = keys.GetSelectorPrefix(WeakFingerprint);
+
+        // A listing can contain keys written by a different version of the plugin, so these must not throw.
+        Assert.IsFalse(keys.TryParseSelector(prefix, WeakFingerprint, out _), "Empty remainder.");
+        Assert.IsFalse(keys.TryParseSelector($"{prefix}00", WeakFingerprint, out _), "Missing the strong fingerprint.");
+        Assert.IsFalse(keys.TryParseSelector($"{prefix}not-a-hash/00", WeakFingerprint, out _), "Unparsable PathSet hash.");
+        Assert.IsFalse(keys.TryParseSelector($"{prefix}a/b/c", WeakFingerprint, out _), "Too many segments.");
+
+        string pathSetHash = keys.GetSelector(CreateStrongFingerprint(new byte[] { 42 })).Substring(prefix.Length).Split('/')[0];
+        Assert.IsFalse(keys.TryParseSelector($"{prefix}{pathSetHash}/zz", WeakFingerprint, out _), "Unparsable strong fingerprint.");
+    }
+}