Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Stress/perf automation #1077

Open
wants to merge 11 commits into
base: main
Choose a base branch
from
Open

Stress/perf automation #1077

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
ca91156
Stress/perf test automation: devops yaml, fault injection via Windows
CIPop Sep 17, 2019
1ff42ac
Adding success rate KPI, redirect stderr.
CIPop Oct 1, 2019
c964116
Error reporting.
CIPop Oct 1, 2019
1f7918b
KPI always displayed.
CIPop Oct 1, 2019
a133e06
KPI condition fix.
CIPop Oct 1, 2019
ed73b11
Write-Error crashes the process.
CIPop Oct 1, 2019
1e955b4
Filter final error report.
CIPop Oct 1, 2019
b9af217
Typo in error file match
CIPop Oct 1, 2019
51d6883
Separate KPIs for service and device success rate.
CIPop Oct 1, 2019
e5aa9ea
Fixing test method name.
CIPop Oct 1, 2019
fac5e9e
Catching all exceptions in Main to ensure the log is properly flushed…
CIPop Oct 2, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 42 additions & 0 deletions e2e/stress/IoTClientPerf/Configuration.Stress.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System;
using System.Globalization;
using System.Security.Cryptography.X509Certificates;

namespace Microsoft.Azure.Devices.E2ETests
Expand All @@ -25,6 +26,41 @@ public static partial class Stress

private static Lazy<X509Certificate2> s_cert = new Lazy<X509Certificate2>(() => { return Configuration.IoTHub.GetCertificateWithPrivateKey(); });

/// <summary>
/// Gets the import export BLOB URI.
/// </summary>
public static string ImportExportBlobUri => GetValue("IOTHUB_IMPORTEXPORT_BLOB_URI");

/// <summary>
/// Gets the connected devices percentage expected by the runner after the test ended.
/// </summary>
public static long? ConnectedDevicesPercentage => ParseNullable(GetValue("IOTHUB_PERF_CONNECTED_PERCENTAGE", ""));

/// <summary>
/// Gets the connected devices percentage expected by the runner after the test ended.
/// </summary>
public static long? TcpConnectionsPercentage => ParseNullable(GetValue("IOTHUB_PERF_TCP_PERCENTAGE", ""));

/// <summary>
/// Gets the requests per second minimum average after the test ended.
/// </summary>
public static long? RequestsPerSecondMinAvg => ParseNullable(GetValue("IOTHUB_PERF_RPS_MIN_AVG", ""));

/// <summary>
/// Gets the requests per second minimum standard deviation after the test ended.
/// </summary>
public static long? RequestsPerSecondMaxStd => ParseNullable(GetValue("IOTHUB_PERF_RPS_MAX_STD", ""));

/// <summary>
/// Gets the requests per second minimum standard deviation after the test ended.
/// </summary>
public static long? GCMemoryBytes => ParseNullable(GetValue("IOTHUB_PERF_GC_MEM_BYTES_MAX", ""));

/// <summary>
/// Success rate defined as operations completed / (completed + failed + cancelled).
/// </summary>
public static long? SuccessRate => ParseNullable(GetValue("IOTHUB_PERF_SUCCESS_RATE_PERCENTAGE", ""));

public static string GetDeviceNameById(int id, string authType)
{
return $"{NamePrefix}_{authType}_{id}";
Expand All @@ -43,6 +79,12 @@ public static string GetConnectionStringById(int id, string authType)
public static string Key2 => s_key2.Value;

public static X509Certificate2 Certificate => s_cert.Value;

private static long? ParseNullable(string s)
{
if (long.TryParse(s, out long l)) return l;
return null;
}
}
}
}
22 changes: 15 additions & 7 deletions e2e/stress/IoTClientPerf/IoTClientPerf.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,27 @@

<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp2.1</TargetFramework>
<TargetFramework>netcoreapp2.2</TargetFramework>
<RootNamespace>Microsoft.Azure.Devices.E2ETests</RootNamespace>
<RootDir>$(MSBuildProjectDirectory)\..\..\..</RootDir>
<CommonTest>$(RootDir)\common\test</CommonTest>
</PropertyGroup>

<ItemGroup>
<Compile Include="..\..\..\common\test\Configuration.cs" Link="Configuration.cs" />
<Compile Include="..\..\..\common\test\Configuration.IoTHub.cs" Link="Configuration.IoTHub.cs" />
<Compile Include="$(CommonTest)\Configuration.cs" Link="Configuration.cs" />
<Compile Include="$(CommonTest)\Configuration.IoTHub.cs" Link="Configuration.IoTHub.cs" />
</ItemGroup>

<ItemGroup>
<ProjectReference Include="..\..\..\iothub\device\src\Microsoft.Azure.Devices.Client.csproj" />
<ProjectReference Include="..\..\..\iothub\service\src\Microsoft.Azure.Devices.csproj" />
<ProjectReference Include="..\..\..\shared\src\Microsoft.Azure.Devices.Shared.csproj" />
<ItemGroup Condition=" '$(AZURE_IOT_LOCALPACKAGES)' == '' ">
<ProjectReference Include="$(RootDir)\iothub\device\src\Microsoft.Azure.Devices.Client.csproj" />
<ProjectReference Include="$(RootDir)\iothub\service\src\Microsoft.Azure.Devices.csproj" />
<ProjectReference Include="$(RootDir)\shared\src\Microsoft.Azure.Devices.Shared.csproj" />
</ItemGroup>

<ItemGroup Condition=" '$(AZURE_IOT_LOCALPACKAGES)' != '' ">
<PackageReference Include="Microsoft.Azure.Devices" Version="1.*" />
<PackageReference Include="Microsoft.Azure.Devices.Shared" Version="1.*" />
<PackageReference Include="Microsoft.Azure.Devices.Client" Version="1.*" />
</ItemGroup>

</Project>
46 changes: 40 additions & 6 deletions e2e/stress/IoTClientPerf/ParallelRun.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Runtime.ExceptionServices;
using System.Threading;
using System.Threading.Tasks;

Expand Down Expand Up @@ -43,10 +45,15 @@ public ParallelRun(

public async Task RunAsync(bool runOnce, CancellationToken ct)
{
int cursor_left, cursor_top;
cursor_left = Console.CursorLeft;
cursor_top = Console.CursorTop;
int cursor_left = 0, cursor_top = 0;

try
{
cursor_left = Console.CursorLeft;
cursor_top = Console.CursorTop;
}
catch (IOException) { /* Avoid "The handle is invalid" exception in DevOps */ }

int actualParallel = Math.Min(_parallelOperations, _tests.Length);
int currentInstance = 0;

Expand Down Expand Up @@ -82,6 +89,14 @@ public async Task RunAsync(bool runOnce, CancellationToken ct)
break;
case TaskStatus.Faulted:
statInterimFaulted++;
foreach (Exception ex in finished.Exception.InnerExceptions)
{
if (ex is ParallelRunFatalException)
{
// Crash the process to simplify analysis. Recover original stack.
((ParallelRunFatalException)ex).ThrowInner();
}
}
break;
case TaskStatus.RanToCompletion:
statInterimCompleted++;
Expand All @@ -103,9 +118,13 @@ public async Task RunAsync(bool runOnce, CancellationToken ct)
double statInterimSeconds = statInterimSw.Elapsed.TotalSeconds;
statTotalCompleted += statInterimCompleted;

Console.SetCursorPosition(cursor_left, cursor_top);
cursor_left = Console.CursorLeft;
cursor_top = Console.CursorTop;
try
{
Console.SetCursorPosition(cursor_left, cursor_top);
cursor_left = Console.CursorLeft;
cursor_top = Console.CursorTop;
}
catch (IOException) { /* Avoid "The handle is invalid" exception in DevOps */ }

_updateStatistics(statInterimCompleted, statInterimFaulted, statInterimCancelled, statInterimSeconds);
if (drain) Console.Write("Waiting for tasks to finish...\r");
Expand Down Expand Up @@ -142,4 +161,19 @@ public async Task RunAsync(bool runOnce, CancellationToken ct)
}
}
}

public class ParallelRunFatalException : Exception
{
private ExceptionDispatchInfo _exceptionDispatchInfo;

public ParallelRunFatalException(ExceptionDispatchInfo innerExceptionDispatchInfo)
{
_exceptionDispatchInfo = innerExceptionDispatchInfo;
}

public void ThrowInner()
{
_exceptionDispatchInfo.Throw();
}
}
}
Loading