Skip to content

Commit 7b8c420

Browse files
authored
Cert rotator (#4617)
* WIP * Add certificate handling for auth * Add cert handling for upload.py * Remove certdeploy * Remove old constants * Add CertHelper code and build step * Move build step for CertHelper * Address PR feedback * Remove islinux * Add CertRotator to CertHelper * WIP * Fix improper list() usage * Yaml testing changes * Make RunCommand verbose * Testing * Add feature to RunCommand to not echo stdout * Remove pragma and fixup tests * Update moq version * Remove testing changes * Remove a few more testing changes * Remove last testing change * Add simple logging * Address pr feedback * Address PR feedback
1 parent 0db4c9b commit 7b8c420

15 files changed

+696
-21
lines changed

scripts/performance/common.py

+14-1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
import os
1818
import sys
1919
import time
20+
import base64
2021
from typing import Callable, List, Optional, Tuple, Type, TypeVar
2122

2223

@@ -138,6 +139,10 @@ def get_packages_directory() -> str:
138139
'''
139140
return os.path.join(get_artifacts_directory(), 'packages')
140141

142+
def base64_to_bytes(base64_string: str) -> bytes:
143+
byte_data = base64.b64decode(base64_string)
144+
return byte_data
145+
141146
@contextmanager
142147
def push_dir(path: Optional[str] = None):
143148
'''
@@ -233,6 +238,7 @@ def __init__(
233238
cmdline: List[str],
234239
success_exit_codes: Optional[List[int]] = None,
235240
verbose: bool = False,
241+
echo: bool = True,
236242
retry: int = 0):
237243
if cmdline is None:
238244
raise TypeError('Unspecified command line to be executed.')
@@ -242,6 +248,7 @@ def __init__(
242248
self.__cmdline = cmdline
243249
self.__verbose = verbose
244250
self.__retry = retry
251+
self.__echo = echo
245252

246253
if success_exit_codes is None:
247254
self.__success_exit_codes = [0]
@@ -261,6 +268,11 @@ def success_exit_codes(self) -> List[int]:
261268
'''
262269
return self.__success_exit_codes
263270

271+
@property
272+
def echo(self) -> bool:
273+
'''Enables/Disables echoing of STDOUT'''
274+
return self.__echo
275+
264276
@property
265277
def verbose(self) -> bool:
266278
'''Enables/Disables verbosity.'''
@@ -296,7 +308,8 @@ def __runinternal(self, working_directory: Optional[str] = None) -> Tuple[int, s
296308
line = raw_line.decode('utf-8', errors='backslashreplace')
297309
self.__stdout.write(line)
298310
line = line.rstrip()
299-
getLogger().info(line)
311+
if self.echo:
312+
getLogger().info(line)
300313
proc.wait()
301314
return (proc.returncode, quoted_cmdline)
302315

scripts/performance/constants.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,5 @@
55
UPLOAD_STORAGE_URI = 'https://pvscmdupload.{}.core.windows.net'
66
UPLOAD_QUEUE = 'resultsqueue'
77
TENANT_ID = '72f988bf-86f1-41af-91ab-2d7cd011db47'
8-
CLIENT_ID = 'a231f733-103b-46e9-b58a-9416edde0eb4'
8+
ARC_CLIENT_ID = 'a231f733-103b-46e9-b58a-9416edde0eb4'
9+
CERT_CLIENT_ID = '8c4b65ef-5a73-4d5a-a298-962d4a4ef7bc'

scripts/run_performance_job.py

+26-11
Original file line numberDiff line numberDiff line change
@@ -774,6 +774,32 @@ def run_performance_job(args: RunPerformanceJobArgs):
774774
getLogger().info("Copying global.json to payload directory")
775775
shutil.copy(os.path.join(args.performance_repo_dir, 'global.json'), os.path.join(performance_payload_dir, 'global.json'))
776776

777+
# Building CertHelper needs to happen here as we need it on every run. This also means that we will need to move the calculation
778+
# of the parameters needed outside of the if block
779+
780+
framework = os.environ["PERFLAB_Framework"]
781+
os.environ["PERFLAB_TARGET_FRAMEWORKS"] = framework
782+
if args.os_group == "windows":
783+
runtime_id = f"win-{args.architecture}"
784+
elif args.os_group == "osx":
785+
runtime_id = f"osx-{args.architecture}"
786+
else:
787+
runtime_id = f"linux-{args.architecture}"
788+
789+
dotnet_executable_path = os.path.join(ci_setup_arguments.install_dir, "dotnet")
790+
791+
RunCommand([
792+
dotnet_executable_path, "publish",
793+
"-c", "Release",
794+
"-o", os.path.join(payload_dir, "certhelper"),
795+
"-f", framework,
796+
"-r", runtime_id,
797+
"--self-contained",
798+
os.path.join(args.performance_repo_dir, "src", "tools", "CertHelper", "CertHelper.csproj"),
799+
f"/bl:{os.path.join(args.performance_repo_dir, 'artifacts', 'log', build_config, 'CertHelper.binlog')}",
800+
"-p:DisableTransitiveFrameworkReferenceDownloads=true"],
801+
verbose=True).run()
802+
777803
if args.is_scenario:
778804
set_environment_variable("DOTNET_ROOT", ci_setup_arguments.install_dir, save_to_pipeline=True)
779805
getLogger().info(f"Set DOTNET_ROOT to {ci_setup_arguments.install_dir}")
@@ -782,17 +808,6 @@ def run_performance_job(args: RunPerformanceJobArgs):
782808
set_environment_variable("PATH", new_path, save_to_pipeline=True)
783809
getLogger().info(f"Set PATH to {new_path}")
784810

785-
framework = os.environ["PERFLAB_Framework"]
786-
os.environ["PERFLAB_TARGET_FRAMEWORKS"] = framework
787-
if args.os_group == "windows":
788-
runtime_id = f"win-{args.architecture}"
789-
elif args.os_group == "osx":
790-
runtime_id = f"osx-{args.architecture}"
791-
else:
792-
runtime_id = f"linux-{args.architecture}"
793-
794-
dotnet_executable_path = os.path.join(ci_setup_arguments.install_dir, "dotnet")
795-
796811
os.environ["MSBUILDDISABLENODEREUSE"] = "1" # without this, MSbuild will be kept alive
797812

798813
# build Startup

scripts/upload.py

+19-8
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,11 @@
33
from azure.storage.blob import BlobClient, ContentSettings
44
from azure.storage.queue import QueueClient, TextBase64EncodePolicy
55
from azure.core.exceptions import ResourceExistsError, ClientAuthenticationError
6-
from azure.identity import DefaultAzureCredential, ClientAssertionCredential
6+
from azure.identity import DefaultAzureCredential, ClientAssertionCredential, CertificateCredential
77
from traceback import format_exc
88
from glob import glob
9-
from performance.common import retry_on_exception
10-
from performance.constants import TENANT_ID, CLIENT_ID
9+
from performance.common import retry_on_exception, RunCommand, helixpayload, base64_to_bytes, extension
10+
from performance.constants import TENANT_ID, ARC_CLIENT_ID, CERT_CLIENT_ID
1111
import os
1212
import json
1313

@@ -32,14 +32,25 @@ def upload(globpath: str, container: str, queue: str, sas_token_env: str, storag
3232
credential = None
3333
try:
3434
dac = DefaultAzureCredential()
35-
credential = ClientAssertionCredential(TENANT_ID, CLIENT_ID, lambda: dac.get_token("api://AzureADTokenExchange/.default").token)
35+
credential = ClientAssertionCredential(TENANT_ID, ARC_CLIENT_ID, lambda: dac.get_token("api://AzureADTokenExchange/.default").token)
3636
credential.get_token("https://storage.azure.com/.default")
3737
except ClientAuthenticationError as ex:
38-
getLogger().info("Unable to use managed identity. Falling back to environment variable.")
39-
credential = os.getenv(sas_token_env)
38+
credential = None
39+
getLogger().info("Unable to use managed identity. Falling back to certificate.")
40+
cmd_line = [(os.path.join(str(helixpayload()), 'certhelper', "CertHelper%s" % extension()))]
41+
cert_helper = RunCommand(cmd_line, None, True, False, 0)
42+
cert_helper.run()
43+
for cert in cert_helper.stdout.splitlines():
44+
credential = CertificateCredential(TENANT_ID, CERT_CLIENT_ID, certificate_data=base64_to_bytes(cert))
45+
try:
46+
credential.get_token("https://storage.azure.com/.default")
47+
except ClientAuthenticationError as ex:
48+
credential = None
49+
continue
4050
if credential is None:
41-
getLogger().error("Sas token environment variable {} was not defined.".format(sas_token_env))
42-
return 1
51+
getLogger().error("Unable to authenticate with managed identity or certificates.")
52+
getLogger().info("Falling back to environment variable.")
53+
credential = os.getenv(sas_token_env)
4354

4455
files = glob(globpath, recursive=True)
4556
any_upload_or_queue_failed = False

src/tools/CertHelper/AssemblyInfo.cs

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Linq;
4+
using System.Runtime.CompilerServices;
5+
using System.Text;
6+
using System.Threading.Tasks;
7+
8+
[assembly: InternalsVisibleTo("CertHelperTests")]
9+
namespace CertHelper;
10+
internal class AssemblyInfo
11+
{
12+
}
+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<OutputType>Exe</OutputType>
5+
<TargetFramework>$(PERFLAB_TARGET_FRAMEWORKS)</TargetFramework>
6+
<!-- Supported target frameworks -->
7+
<TargetFramework Condition="'$(TargetFramework)' == ''">net9.0</TargetFramework>
8+
<ImplicitUsings>enable</ImplicitUsings>
9+
<Nullable>enable</Nullable>
10+
</PropertyGroup>
11+
12+
<ItemGroup>
13+
<PackageReference Include="Azure.Core" Version="1.44.1" />
14+
<PackageReference Include="Azure.Identity" Version="1.11.4" />
15+
<PackageReference Include="Azure.Security.KeyVault.Certificates" Version="4.7.0" />
16+
<PackageReference Include="Azure.Security.KeyVault.Secrets" Version="4.7.0" />
17+
<PackageReference Include="Azure.Storage.Blobs" Version="12.23.0" />
18+
</ItemGroup>
19+
20+
</Project>

src/tools/CertHelper/CertHelper.sln

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
2+
Microsoft Visual Studio Solution File, Format Version 12.00
3+
# Visual Studio Version 17
4+
VisualStudioVersion = 17.12.35514.174 d17.12
5+
MinimumVisualStudioVersion = 10.0.40219.1
6+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CertHelper", "CertHelper.csproj", "{165A37BD-2E9E-4D0A-8402-BB58C29A0BF4}"
7+
EndProject
8+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CertRotatorTests", "..\CertHelperTests\CertRotatorTests.csproj", "{AEA0F93B-EC9B-4438-991E-A80C0C82B3D1}"
9+
EndProject
10+
Global
11+
GlobalSection(SolutionConfigurationPlatforms) = preSolution
12+
Debug|Any CPU = Debug|Any CPU
13+
Release|Any CPU = Release|Any CPU
14+
EndGlobalSection
15+
GlobalSection(ProjectConfigurationPlatforms) = postSolution
16+
{165A37BD-2E9E-4D0A-8402-BB58C29A0BF4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
17+
{165A37BD-2E9E-4D0A-8402-BB58C29A0BF4}.Debug|Any CPU.Build.0 = Debug|Any CPU
18+
{165A37BD-2E9E-4D0A-8402-BB58C29A0BF4}.Release|Any CPU.ActiveCfg = Release|Any CPU
19+
{165A37BD-2E9E-4D0A-8402-BB58C29A0BF4}.Release|Any CPU.Build.0 = Release|Any CPU
20+
{AEA0F93B-EC9B-4438-991E-A80C0C82B3D1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
21+
{AEA0F93B-EC9B-4438-991E-A80C0C82B3D1}.Debug|Any CPU.Build.0 = Debug|Any CPU
22+
{AEA0F93B-EC9B-4438-991E-A80C0C82B3D1}.Release|Any CPU.ActiveCfg = Release|Any CPU
23+
{AEA0F93B-EC9B-4438-991E-A80C0C82B3D1}.Release|Any CPU.Build.0 = Release|Any CPU
24+
EndGlobalSection
25+
GlobalSection(SolutionProperties) = preSolution
26+
HideSolutionNode = FALSE
27+
EndGlobalSection
28+
EndGlobal

src/tools/CertHelper/Constants.cs

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Linq;
4+
using System.Text;
5+
using System.Threading.Tasks;
6+
7+
namespace CertHelper;
8+
public class Constants
9+
{
10+
public static readonly string Cert1Name = "LabCert1";
11+
public static readonly string Cert2Name = "LabCert2";
12+
public static readonly Uri Cert1Id = new Uri("https://test.vault.azure.net/certificates/LabCert1/07a7d98bf4884e5c40e690e02b96b3b4");
13+
public static readonly Uri Cert2Id = new Uri("https://test.vault.azure.net/certificates/LabCert2/07a7d98bf4884e5c41e690e02b96b3b4");
14+
}

src/tools/CertHelper/IX509Store.cs

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Linq;
4+
using System.Security.Cryptography.X509Certificates;
5+
using System.Text;
6+
using System.Threading.Tasks;
7+
8+
namespace CertHelper;
9+
public interface IX509Store
10+
{
11+
X509Certificate2Collection Certificates { get; }
12+
string? Name { get; }
13+
StoreLocation Location { get; }
14+
X509Store GetX509Store();
15+
}
16+
17+
public class TestableX509Store : IX509Store
18+
{
19+
public X509Certificate2Collection Certificates { get => store.Certificates; }
20+
21+
public string? Name => store.Name;
22+
23+
public StoreLocation Location => store.Location;
24+
25+
private X509Store store;
26+
public TestableX509Store(OpenFlags flags = OpenFlags.ReadOnly)
27+
{
28+
store = new X509Store(StoreName.My, StoreLocation.CurrentUser, flags);
29+
}
30+
31+
public X509Store GetX509Store()
32+
{
33+
return store;
34+
}
35+
}

src/tools/CertHelper/KeyVaultCert.cs

+110
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
using Azure;
2+
using Azure.Core;
3+
using Azure.Identity;
4+
using Azure.Security.KeyVault.Certificates;
5+
using Azure.Security.KeyVault.Secrets;
6+
using System;
7+
using System.Collections.Generic;
8+
using System.Linq;
9+
using System.Runtime.CompilerServices;
10+
using System.Security.Cryptography.X509Certificates;
11+
using System.Text;
12+
using System.Threading.Tasks;
13+
14+
namespace CertHelper;
15+
16+
public class KeyVaultCert
17+
{
18+
private readonly string _keyVaultUrl = "https://dotnetperfkeyvault.vault.azure.net/";
19+
private readonly string _tenantId = "72f988bf-86f1-41af-91ab-2d7cd011db47";
20+
private readonly string _clientId = "8c4b65ef-5a73-4d5a-a298-962d4a4ef7bc";
21+
22+
public X509Certificate2Collection KeyVaultCertificates { get; set; }
23+
public ILocalCert LocalCerts { get; set; }
24+
private TokenCredential _credential { get; set; }
25+
private CertificateClient _certClient { get; set; }
26+
private SecretClient _secretClient { get; set; }
27+
28+
public KeyVaultCert(TokenCredential? cred = null, CertificateClient? certClient = null, SecretClient? secretClient = null, ILocalCert? localCerts = null)
29+
{
30+
LocalCerts = localCerts ?? new LocalCert();
31+
_credential = cred ?? GetCertifcateCredentialAsync(_tenantId, _clientId, LocalCerts.Certificates).Result;
32+
_certClient = certClient ?? new CertificateClient(new Uri(_keyVaultUrl), _credential);
33+
_secretClient = secretClient ?? new SecretClient(new Uri(_keyVaultUrl), _credential);
34+
KeyVaultCertificates = new X509Certificate2Collection();
35+
}
36+
37+
public async Task LoadKeyVaultCertsAsync()
38+
{
39+
KeyVaultCertificates.Add(await FindCertificateInKeyVaultAsync(Constants.Cert1Name));
40+
KeyVaultCertificates.Add(await FindCertificateInKeyVaultAsync(Constants.Cert2Name));
41+
42+
if (KeyVaultCertificates.Where(c => c == null).Count() > 0)
43+
{
44+
throw new Exception("One or more certificates not found");
45+
}
46+
}
47+
48+
private async Task<ClientCertificateCredential> GetCertifcateCredentialAsync(string tenantId, string clientId, X509Certificate2Collection certCollection)
49+
{
50+
ClientCertificateCredential? ccc = null;
51+
Exception? exception = null;
52+
foreach (var cert in certCollection)
53+
{
54+
try
55+
{
56+
ccc = new ClientCertificateCredential(tenantId, clientId, cert);
57+
await ccc.GetTokenAsync(new TokenRequestContext(new string[] { "https://vault.azure.net/.default" }));
58+
break;
59+
}
60+
catch (Exception ex)
61+
{
62+
ccc = null;
63+
exception = ex;
64+
}
65+
}
66+
if(ccc == null)
67+
{
68+
throw new Exception("Both certificates failed to authenticate", exception);
69+
}
70+
return ccc;
71+
}
72+
73+
private async Task<X509Certificate2> FindCertificateInKeyVaultAsync(string certName)
74+
{
75+
var keyVaultCert = await _certClient.GetCertificateAsync(certName);
76+
if(keyVaultCert.Value == null)
77+
{
78+
throw new Exception("Certificate not found in Key Vault");
79+
}
80+
var secret = await _secretClient.GetSecretAsync(keyVaultCert.Value.Name, keyVaultCert.Value.SecretId.Segments.Last());
81+
if(secret.Value == null)
82+
{
83+
throw new Exception("Certificate secret not found in Key Vault");
84+
}
85+
var certBytes = Convert.FromBase64String(secret.Value.Value);
86+
#if NET9_0_OR_GREATER
87+
var cert = X509CertificateLoader.LoadPkcs12(certBytes, "", X509KeyStorageFlags.Exportable);
88+
#else
89+
var cert = new X509Certificate2(certBytes, "", X509KeyStorageFlags.Exportable);
90+
#endif
91+
return cert;
92+
}
93+
94+
public bool ShouldRotateCerts()
95+
{
96+
var keyVaultThumbprints = new HashSet<string>();
97+
foreach (var cert in KeyVaultCertificates)
98+
{
99+
keyVaultThumbprints.Add(cert.Thumbprint);
100+
}
101+
foreach(var cert in LocalCerts.Certificates)
102+
{
103+
if (!keyVaultThumbprints.Contains(cert.Thumbprint))
104+
{
105+
return true;
106+
}
107+
}
108+
return false;
109+
}
110+
}

0 commit comments

Comments
 (0)