Skip to content

Commit

Permalink
-Improve integration test reliability
Browse files Browse the repository at this point in the history 
-Improve Get-SensorHistory unit test reliability when real time time differences cause one second errors in expected vs actual API request URLs
-Fix Uninstall-GoPrtgServer tests failing on Linux PowerShell Core due to new error message when command is not found
  • Loading branch information
@lordmilko
lordmilko committed Nov 25, 2020
1 parent 2dd5350 commit 0ac0346
Show file tree
Hide file tree
Showing 16 changed files with 228 additions and 59 deletions.
37 changes: 29 additions & 8 deletions src/PrtgAPI.Tests.IntegrationTests/CSharp/Infrastructure/PrtgClientConnectionTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
using System.Threading.Tasks;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using PrtgAPI.Parameters;
using PrtgAPI.Tests.IntegrationTests.Support;
using PrtgAPI.Tests.UnitTests.Support;

namespace PrtgAPI.Tests.IntegrationTests.Infrastructure
Expand Down Expand Up @@ -223,18 +224,18 @@ private void Logic_Client_RetryRequestInternal(Action<PrtgClient> action, bool i
var localClient = new PrtgClient(Settings.ServerWithProto, Settings.UserName, Settings.Password);
localClient.RetryRequest += (sender, args) =>
{
Logger.LogTestDetail($"Handling retry {retriesMade + 1}");
retriesMade++;

Logger.LogTestDetail($"Handling retry {retriesMade}");

if (!isAsync)
AssertEx.AreEqual(initialThread, Thread.CurrentThread.ManagedThreadId, "Event was not handled on initial thread");
retriesMade++;
};
localClient.RetryCount = retriesToMake;

Logger.LogTestDetail("Stopping PRTG Service");

coreService.Stop();
coreService.WaitForStatus(ServiceControllerStatus.Stopped);
coreService.StopAndWaitForStatus(ServiceControllerStatus.Stopped);

try
{
Expand All @@ -251,11 +252,31 @@ private void Logic_Client_RetryRequestInternal(Action<PrtgClient> action, bool i
finally
{
Logger.LogTestDetail("Starting PRTG Service");
coreService.Start();
coreService.WaitForStatus(ServiceControllerStatus.Running);
coreService.StartAndWaitForStatus(ServiceControllerStatus.Running);

var time = 90;

while (true)
{
try
{
var probe = client.GetProbes();

break;
}
catch (HttpRequestException)
{
var duration = 5;

time -= duration;

if (time < 0)
throw;

Logger.LogTestDetail("Sleeping for 30 seconds");
Thread.Sleep(30000);
Logger.LogTestDetail($"Sleeping for {duration} seconds as service is not up");
Thread.Sleep(duration * 1000);
}
}

Logger.LogTestDetail("Refreshing and sleeping for 20 seconds");
localClient.RefreshObject(Settings.Device);
Expand Down
38 changes: 23 additions & 15 deletions src/PrtgAPI.Tests.IntegrationTests/CSharp/ObjectData/Query/FilterTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1145,11 +1145,13 @@ public void Data_QueryFilter_SensorProperties_Downtime_NotEquals()

[TestMethod]
[IntegrationTest]
public void Data_QueryFilter_SensorProperties_Downtime_GreaterThan()
public void Data_QueryFilter_SensorProperties_Downtime_GreaterThan_bad()
{
var downSensor = client.GetSensor(Settings.DownSensor);

ExecuteSensor(s => s.Downtime > downSensor.Downtime, Property.Downtime, downSensor.Downtime, FilterOperator.GreaterThan);
var expectedDowntime = downSensor.Downtime - 10;

ExecuteSensor(s => s.Downtime > expectedDowntime, Property.Downtime, expectedDowntime, FilterOperator.GreaterThan);
}

[TestMethod]
Expand Down Expand Up @@ -1534,7 +1536,7 @@ public void Data_QueryFilter_SensorProperties_TotalMonitorTime_Equals()
{
var monitorTime = client.GetSensor(Settings.UpSensor).TotalMonitorTime;

Assert.IsNotNull(monitorTime);
Assert.IsTrue(monitorTime.TotalSeconds >= 0, "Cannot test TotalMonitorTime if TotalSeconds value is 0");

ExecuteSensor(s => s.TotalMonitorTime == monitorTime, Property.TotalMonitorTime, monitorTime);
}
Expand All @@ -1545,7 +1547,7 @@ public void Data_QueryFilter_SensorProperties_TotalMonitorTime_NotEquals()
{
var monitorTime = client.GetSensor(Settings.UpSensor).TotalMonitorTime;

Assert.IsNotNull(monitorTime);
Assert.IsTrue(monitorTime.TotalSeconds >= 0, "Cannot test TotalMonitorTime if TotalSeconds value is 0");

ExecuteSensor(s => s.TotalMonitorTime != monitorTime, Property.TotalMonitorTime, monitorTime, FilterOperator.NotEquals);
}
Expand All @@ -1554,9 +1556,9 @@ public void Data_QueryFilter_SensorProperties_TotalMonitorTime_NotEquals()
[IntegrationTest]
public void Data_QueryFilter_SensorProperties_TotalMonitorTime_GreaterThan()
{
var monitorTime = client.GetSensor(Settings.UpSensor).TotalMonitorTime;
var monitorTime = client.GetSensor(Settings.UpSensor).TotalMonitorTime.Subtract(TimeSpan.FromSeconds(10));

Assert.IsNotNull(monitorTime);
Assert.IsTrue(monitorTime.TotalSeconds >= 0, "Cannot test TotalMonitorTime if TotalSeconds value is 0");

ExecuteSensor(s => s.TotalMonitorTime > monitorTime, Property.TotalMonitorTime, monitorTime, FilterOperator.GreaterThan);
}
Expand All @@ -1565,9 +1567,9 @@ public void Data_QueryFilter_SensorProperties_TotalMonitorTime_GreaterThan()
[IntegrationTest]
public void Data_QueryFilter_SensorProperties_TotalMonitorTime_LessThan()
{
var monitorTime = client.GetSensor(Settings.UpSensor).TotalMonitorTime;
var monitorTime = client.GetSensor(Settings.UpSensor).TotalMonitorTime.Add(TimeSpan.FromSeconds(10));

Assert.IsNotNull(monitorTime);
Assert.IsTrue(monitorTime.TotalSeconds >= 10, "Cannot test TotalMonitorTime if TotalSeconds value is 0");

ExecuteSensor(s => s.TotalMonitorTime < monitorTime, Property.TotalMonitorTime, monitorTime, FilterOperator.LessThan);
}
Expand All @@ -1578,7 +1580,7 @@ public void Data_QueryFilter_SensorProperties_TotalMonitorTime_Contains()
{
var monitorTime = client.GetSensor(Settings.UpSensor).TotalMonitorTime;

Assert.IsNotNull(monitorTime);
Assert.IsTrue(monitorTime.TotalSeconds >= 0, "Cannot test TotalMonitorTime if TotalSeconds value is 0");

ExecuteSensor(s => s.TotalMonitorTime.ToString().Contains(monitorTime.ToString()), Property.TotalMonitorTime, monitorTime);
}
Expand Down Expand Up @@ -1776,9 +1778,12 @@ public void Data_QueryFilter_SensorProperties_Uptime_Contains()
[IntegrationTest]
public void Data_QueryFilter_DeviceProperties_Condition_Equals()
{
PrepareCondition(device =>
Retry(retry =>
{
ExecuteDevice(d => d.Condition == device.Condition, Property.Condition, device.Condition);
PrepareCondition(device =>
{
ExecuteDevice(d => d.Condition == device.Condition, Property.Condition, device.Condition);
});
});
}

Expand Down Expand Up @@ -1816,9 +1821,12 @@ public void Data_QueryFilter_DeviceProperties_Condition_LessThan()
[IntegrationTest]
public void Data_QueryFilter_DeviceProperties_Condition_Contains()
{
PrepareCondition(device =>
Retry(retry =>
{
ExecuteDevice(d => d.Condition.Contains(device.Condition), Property.Condition, device.Condition, FilterOperator.Contains);
PrepareCondition(device =>
{
ExecuteDevice(d => d.Condition.Contains(device.Condition), Property.Condition, device.Condition, FilterOperator.Contains);
});
});
}

Expand Down Expand Up @@ -2536,13 +2544,13 @@ private void ExecuteObject<T>(
{
if (retry)
{
Assert.IsTrue(linqObjects.Count > 0);
Assert.IsTrue(filterObjects.Count > 0);
Assert.IsTrue(linqObjects.Count > 0);
Assert.IsTrue(queryObjects.Count > 0);
}

AssertEx.IsTrue(filterObjects.Count > 0, "Filter objects did not return any results. Are there even any results that match the specified search criteria?");
AssertEx.IsTrue(linqObjects.Count > 0, "LINQ objects did not return any results");
AssertEx.IsTrue(filterObjects.Count > 0, "Filter objects did not return any results");
AssertEx.IsTrue(queryObjects.Count > 0, "Query objects did not return any results");
}

Expand Down
2 changes: 1 addition & 1 deletion src/PrtgAPI.Tests.IntegrationTests/CSharp/ObjectManipulation/AddObjectTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ private void AddToInvalidObject(int objectId)

AssertEx.Throws<PrtgRequestException>(
() => client.AddSensor(objectId, parameters),
"The desired object cannot be created here"
"object cannot be created here"
);
}

Expand Down
30 changes: 28 additions & 2 deletions src/PrtgAPI.Tests.IntegrationTests/CSharp/ObjectManipulation/StateTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,12 @@ public class PrtgClientStateTests : BasePrtgClientTest
[IntegrationTest]
public void Action_State_AcknowledgeAndResume()
{
AssertEx.AreEqual(Status.Down, GetSensor(Settings.DownSensor).Status, $"Initial sensor status was not {Status.Down}");
var initial = GetSensor(Settings.DownSensor);

if (initial.Status != Status.Down)
ServerManager.RepairConfig();

AssertEx.AreEqual(Status.Down, initial.Status, "Initial status was not down");

var message = "Unit Testing FTW!";

Expand All @@ -34,6 +39,16 @@ public void Action_State_AcknowledgeAndResume()
[IntegrationTest]
public void Action_State_PauseAndResume()
{
var initial = GetSensor(Settings.UpSensor);

if (initial.Status != Status.Up)
{
client.ResumeObject(initial.Id);
ServerManager.RepairConfig();
}

AssertEx.AreEqual(Status.Up, initial.Status, "Initial status was not up");

Logger.LogTestDetail("Pausing object");
client.PauseObject(Settings.UpSensor);
CheckAndSleep(Settings.UpSensor);
Expand Down Expand Up @@ -68,6 +83,12 @@ public void Action_State_PauseForDuration()
{
var initial = GetSensor(Settings.UpSensor);

if (initial.Status != Status.Up)
{
client.ResumeObject(initial.Id);
ServerManager.RepairConfig();
}

AssertEx.AreEqual(Status.Up, initial.Status, "Initial status was not up");

Logger.LogTestDetail("Pausing for 1 minute");
Expand Down Expand Up @@ -111,7 +132,12 @@ public void Action_State_PauseForDuration()
[IntegrationTest]
public void Action_State_SimulateErrorAndResume()
{
AssertEx.AreEqual(Status.Up, GetSensor(Settings.UpSensor).Status, $"Initial sensor state was not {Status.Up}");
var initial = GetSensor(Settings.UpSensor);

if (initial.Status != Status.Up)
ServerManager.RepairConfig();

AssertEx.AreEqual(Status.Up, initial.Status, "Initial status was not up");

Logger.LogTestDetail("Simulating error status");
client.SimulateError(Settings.UpSensor);
Expand Down
2 changes: 1 addition & 1 deletion src/PrtgAPI.Tests.IntegrationTests/PowerShell/ObjectData/Get-SensorTarget.Tests.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ Describe "Get-SensorTarget_IT" -Tag @("PowerShell", "IntegrationTest") {

It "throws retrieving targets for a sensor type unsupported by a device" {

{ $device | Get-SensorTarget -RawType exchangepsbackup } | Should Throw "An unspecified error occurred while trying to resolve sensor targets. Specified sensor type may not be valid on this device"
{ $device | Get-SensorTarget -RawType exchangepsbackup } | Should Throw "Specified sensor type may not be valid on this device"
}

It "throws attempting to resolve sensor targets as a read-only user" {
Expand Down
4 changes: 2 additions & 2 deletions ...Tests/PowerShell/ObjectManipulation/ObjectProperty/Set-ObjectProperty.Container.Tests.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@ Describe "Set-ObjectProperty_Containers_IT" -Tag @("PowerShell", "IntegrationTes
{ SetChild "Location" "410 Terry Ave. North Seattle" "InheritLocation" $false } | Should Throw "410 Terry Ave N, Seattle, WA 98109, United States"

# Location: Coordinates
$lat = 51.4521018
$lon = 13.0766654
$lat = 51.4521
$lon = 13.077

SetValue "Location" "$lat, $lon" $true
$prop = $object | Get-ObjectProperty
Expand Down
4 changes: 2 additions & 2 deletions src/PrtgAPI.Tests.IntegrationTests/PowerShell/ObjectManipulation/Pause-Object.Tests.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ Describe "Pause-Object_IT" -Tag @("PowerShell", "IntegrationTest") {
$pausedSensor = WaitForStatus $pausedSensor Up 90

LogTestDetail "Object should be unpaused. Refreshing object."
$finalSensor = WaitForStatus $pausedSensor Up 30 10
$finalSensor = WaitForStatus $pausedSensor Up 60 5

$finalSensor.Status | Should Be Up
}
Expand All @@ -76,7 +76,7 @@ Describe "Pause-Object_IT" -Tag @("PowerShell", "IntegrationTest") {
$pausedSensor = WaitForStatus $pausedSensor Up 90

LogTestDetail "Object should be unpaused. Refreshing object."
$finalSensor = WaitForStatus $pausedSensor Up 30 10
$finalSensor = WaitForStatus $pausedSensor Up 60 5

$finalSensor.Status | Should Be Up
}
Expand Down
1 change: 1 addition & 0 deletions src/PrtgAPI.Tests.IntegrationTests/PrtgAPI.Tests.IntegrationTests.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@
<Compile Include="CSharp\ObjectManipulation\StateTests.cs" />
<Compile Include="Settings.cs" />
<Compile Include="Settings.Local.cs" Condition="Exists('Settings.Local.cs')" />
<Compile Include="Support\ServiceControllerExtensions.cs" />
<Compile Include="Support\TestCategory\IntegrationTestAttribute.cs" />
</ItemGroup>
<ItemGroup>
Expand Down
16 changes: 6 additions & 10 deletions src/PrtgAPI.Tests.IntegrationTests/Support/ServerManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
using System.Net.NetworkInformation;
using System.ServiceProcess;
using System.Threading;
using PrtgAPI.Tests.IntegrationTests.Support;

namespace PrtgAPI.Tests.IntegrationTests
{
Expand Down Expand Up @@ -253,16 +254,14 @@ private void RemoteInit()
{
Logger.Log("Restoring PRTG Config leftover from previous aborted test");
Logger.LogTest("Stopping PRTGCoreService");
coreService.Stop();
coreService.WaitForStatus(ServiceControllerStatus.Stopped);
coreService.StopAndWaitForStatus(ServiceControllerStatus.Stopped);

Logger.LogTest("Copying PRTG Config");
File.Copy(PrtgConfigBackup, PrtgConfig, true);
File.Delete(PrtgConfigBackup);

Logger.LogTest("Starting PRTGCoreService");
coreService.Start();
coreService.WaitForStatus(ServiceControllerStatus.Running);
coreService.StartAndWaitForStatus(ServiceControllerStatus.Running);

Logger.LogTest("Sleeping for 30 seconds while PRTG starts up");
Thread.Sleep(30 * 1000);
Expand Down Expand Up @@ -330,8 +329,7 @@ private void RemoteCleanup(bool deleteConfig, bool restoreConfig = true)

Logger.Log("Starting service");

coreService.Start();
coreService.WaitForStatus(ServiceControllerStatus.Running);
coreService.StartAndWaitForStatus(ServiceControllerStatus.Running);

if (probeNameNeedsRepairing)
{
Expand All @@ -351,8 +349,7 @@ private void RemoteCleanup(bool deleteConfig, bool restoreConfig = true)
if (probeService.Status != ServiceControllerStatus.Stopped)
Thread.Sleep(5000);

probeService.Start();
probeService.WaitForStatus(ServiceControllerStatus.Running);
probeService.StartAndWaitForStatus(ServiceControllerStatus.Running);
}

Logger.Log("Finished");
Expand Down Expand Up @@ -471,8 +468,7 @@ private void StartService(ServiceController service, string friendlyName, bool r
if (service.Status == ServiceControllerStatus.Stopped)
{
Logger.LogTest($"{service.ServiceName} is not running. Starting service");
service.Start();
service.WaitForStatus(ServiceControllerStatus.Running);
service.StartAndWaitForStatus(ServiceControllerStatus.Running);

Logger.LogTest("Sleeping for 30 seconds while PRTG service starts");
Thread.Sleep(30 * 1000);
Expand Down
Loading

0 comments on commit 0ac0346

Please sign in to comment.