diff --git a/NuGet.config b/NuGet.config index 6e2afbf191..0763a2438a 100644 --- a/NuGet.config +++ b/NuGet.config @@ -8,5 +8,6 @@ + diff --git a/src/WebJobs.Script.WebHost/Metrics/FlexConsumptionMetricsPublisher.cs b/src/WebJobs.Script.WebHost/Metrics/FlexConsumptionMetricsPublisher.cs index 1ccc8478ee..4e6238c65d 100644 --- a/src/WebJobs.Script.WebHost/Metrics/FlexConsumptionMetricsPublisher.cs +++ b/src/WebJobs.Script.WebHost/Metrics/FlexConsumptionMetricsPublisher.cs @@ -26,6 +26,7 @@ public class FlexConsumptionMetricsPublisher : IMetricsPublisher, IDisposable private readonly IHostMetricsProvider _metricsProvider; private readonly object _lock = new object(); private readonly IFileSystem _fileSystem; + private readonly LegionMetricsFileManager _metricsFileManager; private Timer _metricsPublisherTimer; private bool _started = false; @@ -44,6 +45,7 @@ public FlexConsumptionMetricsPublisher(IEnvironment environment, IOptionsMonitor _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _fileSystem = fileSystem ?? new FileSystem(); + _metricsFileManager = new LegionMetricsFileManager(_options.MetricsFilePath, _fileSystem, _logger, _options.MaxFileCount); _metricsProvider = metricsProvider ?? throw new ArgumentNullException(nameof(metricsProvider)); if (_standbyOptions.CurrentValue.InStandbyMode) @@ -66,13 +68,13 @@ public FlexConsumptionMetricsPublisher(IEnvironment environment, IOptionsMonitor internal bool IsAlwaysReady { get; set; } - internal string MetricsFilePath { get; set; } + internal LegionMetricsFileManager MetricsFileManager => _metricsFileManager; public void Start() { Initialize(); - _logger.LogInformation($"Starting metrics publisher (AlwaysReady={IsAlwaysReady}, MetricsPath='{MetricsFilePath}')."); + _logger.LogInformation($"Starting metrics publisher (AlwaysReady={IsAlwaysReady}, MetricsPath='{_metricsFileManager.MetricsFilePath}')."); _metricsPublisherTimer = new Timer(OnFunctionMetricsPublishTimer, null, _initialPublishDelay, _metricPublishInterval); _started = true; @@ -86,7 +88,6 @@ internal void Initialize() _metricPublishInterval = TimeSpan.FromMilliseconds(_options.MetricsPublishIntervalMS); _initialPublishDelay = TimeSpan.FromMilliseconds(_options.InitialPublishDelayMS); _intervalStopwatch = ValueStopwatch.StartNew(); - MetricsFilePath = _options.MetricsFilePath; IsAlwaysReady = _environment.GetEnvironmentVariable(EnvironmentSettingNames.FunctionsAlwaysReadyInstance) == "1"; } @@ -136,7 +137,12 @@ internal async Task OnPublishMetrics(DateTime now) FunctionExecutionTimeMS = FunctionExecutionCount = 0; } - await PublishMetricsAsync(metrics); + await _metricsFileManager.PublishMetricsAsync(metrics); + } + catch (Exception ex) when (!ex.IsFatal()) + { + // ensure no background exceptions escape + _logger.LogError(ex, $"Error publishing metrics."); } finally { @@ -149,84 +155,6 @@ private async void OnFunctionMetricsPublishTimer(object state) await OnPublishMetrics(DateTime.UtcNow); } - private async Task PublishMetricsAsync(Metrics metrics) - { - string fileName = string.Empty; - - try - { - bool metricsPublishEnabled = !string.IsNullOrEmpty(MetricsFilePath); - if (metricsPublishEnabled && !PrepareDirectoryForFile()) - { - return; - } - - string metricsContent = JsonConvert.SerializeObject(metrics); - _logger.PublishingMetrics(metricsContent); - - if (metricsPublishEnabled) - { - fileName = $"{Guid.NewGuid().ToString().ToLower()}.json"; - string filePath = Path.Combine(MetricsFilePath, fileName); - - using (var streamWriter = _fileSystem.File.CreateText(filePath)) - { - await streamWriter.WriteAsync(metricsContent); - } - } - } - catch (Exception ex) when (!ex.IsFatal()) - { - // TODO: consider using a retry strategy here - _logger.LogError(ex, $"Error writing metrics file '{fileName}'."); - } - } - - private bool PrepareDirectoryForFile() - { - if (string.IsNullOrEmpty(MetricsFilePath)) - { - return false; - } - - // ensure the directory exists - _fileSystem.Directory.CreateDirectory(MetricsFilePath); - - var metricsDirectoryInfo = _fileSystem.DirectoryInfo.FromDirectoryName(MetricsFilePath); - var files = metricsDirectoryInfo.GetFiles().OrderBy(p => p.CreationTime).ToList(); - - // ensure we're under the max file count - if (files.Count < _options.MaxFileCount) - { - return true; - } - - // we're at or over limit - // delete enough files that we have space to write a new one - int numToDelete = files.Count - _options.MaxFileCount + 1; - var filesToDelete = files.Take(numToDelete).ToArray(); - - _logger.LogDebug($"Deleting {filesToDelete.Length} metrics file(s)."); - - foreach (var file in filesToDelete) - { - try - { - file.Delete(); - } - catch (Exception ex) when (!ex.IsFatal()) - { - // best effort - _logger.LogError(ex, $"Error deleting metrics file '{file.FullName}'."); - } - } - - files = metricsDirectoryInfo.GetFiles().OrderBy(p => p.CreationTime).ToList(); - - // return true if we have space for a new file - return files.Count < _options.MaxFileCount; - } - private void OnStandbyOptionsChange() { if (!_standbyOptions.CurrentValue.InStandbyMode) diff --git a/src/WebJobs.Script.WebHost/Metrics/LegionMetricsFileManager.cs b/src/WebJobs.Script.WebHost/Metrics/LegionMetricsFileManager.cs new file mode 100644 index 0000000000..7cd0779f14 --- /dev/null +++ b/src/WebJobs.Script.WebHost/Metrics/LegionMetricsFileManager.cs @@ -0,0 +1,109 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.IO; +using System.IO.Abstractions; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Azure.WebJobs.Script.Diagnostics.Extensions; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; + +namespace Microsoft.Azure.WebJobs.Script.WebHost.Metrics +{ + public class LegionMetricsFileManager + { + private readonly IFileSystem _fileSystem; + private readonly int _maxFileCount; + private readonly ILogger _logger; + + public LegionMetricsFileManager(string metricsFilePath, IFileSystem fileSystem, ILogger logger, int maxFileCount) + { + _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + MetricsFilePath = metricsFilePath; + _maxFileCount = maxFileCount; + } + + internal string MetricsFilePath { get; set; } + + private bool PrepareDirectoryForFile() + { + if (string.IsNullOrEmpty(MetricsFilePath)) + { + return false; + } + + // ensure the directory exists + _fileSystem.Directory.CreateDirectory(MetricsFilePath); + + var metricsDirectoryInfo = _fileSystem.DirectoryInfo.FromDirectoryName(MetricsFilePath); + var files = metricsDirectoryInfo.GetFiles().OrderBy(p => p.CreationTime).ToList(); + + // ensure we're under the max file count + if (files.Count < _maxFileCount) + { + return true; + } + + // we're at or over limit + // delete enough files that we have space to write a new one + int numToDelete = files.Count - _maxFileCount + 1; + var filesToDelete = files.Take(numToDelete).ToArray(); + + _logger.LogDebug($"Deleting {filesToDelete.Length} metrics file(s)."); + + foreach (var file in filesToDelete) + { + try + { + file.Delete(); + } + catch (Exception ex) when (!ex.IsFatal()) + { + // best effort + _logger.LogError(ex, $"Error deleting metrics file '{file.FullName}'."); + } + } + + files = metricsDirectoryInfo.GetFiles().OrderBy(p => p.CreationTime).ToList(); + + // return true if we have space for a new file + return files.Count < _maxFileCount; + } + + public async Task PublishMetricsAsync(object metrics) + { + string fileName = string.Empty; + + try + { + bool metricsPublishEnabled = !string.IsNullOrEmpty(MetricsFilePath); + if (metricsPublishEnabled && !PrepareDirectoryForFile()) + { + return; + } + + string metricsContent = JsonConvert.SerializeObject(metrics); + _logger.PublishingMetrics(metricsContent); + + if (metricsPublishEnabled) + { + fileName = $"{Guid.NewGuid().ToString().ToLower()}.json"; + string filePath = Path.Combine(MetricsFilePath, fileName); + + using (var streamWriter = _fileSystem.File.CreateText(filePath)) + { + await streamWriter.WriteAsync(metricsContent); + } + } + } + catch (Exception ex) when (!ex.IsFatal()) + { + // TODO: consider using a retry strategy here + _logger.LogError(ex, $"Error writing metrics file '{fileName}'."); + } + } + } +} \ No newline at end of file diff --git a/src/WebJobs.Script.WebHost/Metrics/LinuxContainerLegionMetricsPublisher.cs b/src/WebJobs.Script.WebHost/Metrics/LinuxContainerLegionMetricsPublisher.cs new file mode 100644 index 0000000000..f0bf03b4bd --- /dev/null +++ b/src/WebJobs.Script.WebHost/Metrics/LinuxContainerLegionMetricsPublisher.cs @@ -0,0 +1,239 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Diagnostics; +using System.IO.Abstractions; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Azure.Functions.Platform.Metrics.LinuxConsumption; +using Microsoft.Azure.WebJobs.Script.Diagnostics; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Microsoft.Azure.WebJobs.Script.WebHost.Metrics +{ + public class LinuxContainerLegionMetricsPublisher : IMetricsPublisher, IDisposable + { + private readonly ILinuxConsumptionMetricsTracker _metricsTracker; + private readonly LegionMetricsFileManager _metricsFileManager; + private readonly TimeSpan _memorySnapshotInterval = TimeSpan.FromMilliseconds(1000); + private readonly TimeSpan _timerStartDelay = TimeSpan.FromSeconds(2); + private readonly IOptionsMonitor _standbyOptions; + private readonly IDisposable _standbyOptionsOnChangeSubscription; + private readonly IEnvironment _environment; + private readonly ILogger _logger; + //private readonly IMetricsLogger _metricsLogger; + private readonly string _containerName; + + private TimeSpan _metricPublishInterval; + private Process _process; + private Timer _processMonitorTimer; + private Timer _metricsPublisherTimer; + private bool _initialized = false; + + public LinuxContainerLegionMetricsPublisher(IEnvironment environment, IOptionsMonitor standbyOptions, ILogger logger, IFileSystem fileSystem, ILinuxConsumptionMetricsTracker metricsTracker, int? metricsPublishIntervalMS = null) + { + _standbyOptions = standbyOptions ?? throw new ArgumentNullException(nameof(standbyOptions)); + _environment = environment ?? throw new ArgumentNullException(nameof(environment)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _metricsTracker = metricsTracker ?? throw new ArgumentNullException(nameof(metricsTracker)); + //_metricsLogger = metricsLogger ?? throw new ArgumentNullException(nameof(metricsLogger)); + _containerName = _environment.GetEnvironmentVariable(EnvironmentSettingNames.ContainerName); + _metricPublishInterval = TimeSpan.FromMilliseconds(metricsPublishIntervalMS ?? 30 * 1000); + + // Default this to 15 minutes worth of files + int maxFileCount = 15 * (int)Math.Ceiling(1.0 * 60 / _metricPublishInterval.TotalSeconds); + string metricsFilePath = _environment.GetEnvironmentVariable(EnvironmentSettingNames.FunctionsMetricsPublishPath); + + _metricsFileManager = new LegionMetricsFileManager(metricsFilePath, fileSystem, logger, maxFileCount); + + _processMonitorTimer = new Timer(OnProcessMonitorTimer, null, Timeout.Infinite, Timeout.Infinite); + _metricsPublisherTimer = new Timer(OnFunctionMetricsPublishTimer, null, Timeout.Infinite, Timeout.Infinite); + + if (_standbyOptions.CurrentValue.InStandbyMode) + { + _standbyOptionsOnChangeSubscription = _standbyOptions.OnChange(o => OnStandbyOptionsChange()); + } + else + { + Start(); + } + } + + public void Initialize() + { + _process = Process.GetCurrentProcess(); + _initialized = true; + } + + public void Start() + { + Initialize(); + + // start the timers by setting the due time + SetTimerInterval(_processMonitorTimer, _timerStartDelay); + SetTimerInterval(_metricsPublisherTimer, _metricPublishInterval); + + _logger.LogInformation(string.Format("Starting metrics publisher for container : {0}.", _containerName)); + } + + private void OnStandbyOptionsChange() + { + if (!_standbyOptions.CurrentValue.InStandbyMode) + { + Start(); + } + } + + public void AddFunctionExecutionActivity(string functionName, string invocationId, int concurrency, string executionStage, bool success, long executionTimeSpan, string executionId, DateTime eventTimeStamp, DateTime functionStartTime) + { + if (!_initialized) + { + return; + } + + Enum.TryParse(executionStage, out FunctionExecutionStage functionExecutionStage); + + FunctionActivity activity = new FunctionActivity + { + FunctionName = functionName, + InvocationId = invocationId, + Concurrency = concurrency, + ExecutionStage = functionExecutionStage, + ExecutionId = executionId, + IsSucceeded = success, + ExecutionTimeSpanInMs = executionTimeSpan, + EventTimeStamp = eventTimeStamp, + StartTime = functionStartTime + }; + + _metricsTracker.AddFunctionActivity(activity); + } + + public void AddMemoryActivity(DateTime timeStampUtc, long data) + { + if (!_initialized) + { + return; + } + + var memoryActivity = new MemoryActivity + { + CommitSizeInBytes = data, + EventTimeStamp = timeStampUtc + }; + + _metricsTracker.AddMemoryActivity(memoryActivity); + } + + private async void OnFunctionMetricsPublishTimer(object state) + { + await OnPublishMetrics(); + } + + internal async Task OnPublishMetrics() + { + try + { + if (_metricsTracker.TryGetMetrics(out LinuxConsumptionMetrics trackedMetrics)) + { + var metricsToPublish = new Metrics + { + FunctionActivity = trackedMetrics.FunctionActivity, + ExecutionCount = trackedMetrics.FunctionExecutionCount, + ExecutionTimeMS = trackedMetrics.FunctionExecutionTimeMS + }; + + await _metricsFileManager.PublishMetricsAsync(metricsToPublish); + } + } + catch (Exception ex) when (!ex.IsFatal()) + { + // ensure no background exceptions escape + _logger.LogError(ex, $"Error publishing metrics."); + } + finally + { + SetTimerInterval(_metricsPublisherTimer, _metricPublishInterval); + } + } + + private void OnProcessMonitorTimer(object state) + { + try + { + _process.Refresh(); + var commitSizeBytes = _process.WorkingSet64; + if (commitSizeBytes != 0) + { + AddMemoryActivity(DateTime.UtcNow, commitSizeBytes); + } + } + catch (Exception e) + { + // throwing this exception will mask other underlying exceptions. + // Log and let other interesting exceptions bubble up. + _logger.LogError(e, nameof(OnProcessMonitorTimer)); + } + finally + { + SetTimerInterval(_processMonitorTimer, _memorySnapshotInterval); + } + } + + private void SetTimerInterval(Timer timer, TimeSpan dueTime) + { + try + { + timer?.Change((int)dueTime.TotalMilliseconds, Timeout.Infinite); + } + catch (ObjectDisposedException) + { + // might race with dispose + } + catch (Exception e) + { + _logger.LogError(e, nameof(SetTimerInterval)); + } + } + + public void OnFunctionStarted(string functionName, string invocationId) + { + // nothing to do + } + + public void OnFunctionCompleted(string functionName, string invocationId) + { + // nothing to do + } + + public void Dispose() + { + _processMonitorTimer?.Dispose(); + _processMonitorTimer = null; + + _metricsPublisherTimer?.Dispose(); + _metricsPublisherTimer = null; + } + + internal class Metrics + { + /// + /// Gets or sets a measure of the function activity for the interval. + /// + public long FunctionActivity { get; set; } + + /// + /// Gets or sets the total execution duration for all functions during this interval. + /// + public long ExecutionTimeMS { get; set; } + + /// + /// Gets or sets the total number of functions invocations that + /// completed during the interval. + /// + public long ExecutionCount { get; set; } + } + } +} \ No newline at end of file diff --git a/src/WebJobs.Script.WebHost/Metrics/LinuxContainerMetricsPublisher.cs b/src/WebJobs.Script.WebHost/Metrics/LinuxContainerMetricsPublisher.cs index 6110fe9df4..69a094478f 100644 --- a/src/WebJobs.Script.WebHost/Metrics/LinuxContainerMetricsPublisher.cs +++ b/src/WebJobs.Script.WebHost/Metrics/LinuxContainerMetricsPublisher.cs @@ -11,8 +11,8 @@ using System.Security.Cryptography.X509Certificates; using System.Threading; using System.Threading.Tasks; +using Microsoft.Azure.Functions.Platform.Metrics.LinuxConsumption; using Microsoft.Azure.WebJobs.Script.Config; -using Microsoft.Azure.WebJobs.Script.WebHost.Models; using Microsoft.Azure.WebJobs.Script.WebHost.Security; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; diff --git a/src/WebJobs.Script.WebHost/Models/ActivityBase.cs b/src/WebJobs.Script.WebHost/Models/ActivityBase.cs deleted file mode 100644 index 02532ef793..0000000000 --- a/src/WebJobs.Script.WebHost/Models/ActivityBase.cs +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System; - -namespace Microsoft.Azure.WebJobs.Script.WebHost.Models -{ - public class ActivityBase - { - public DateTime EventTimeStamp { get; set; } - - public string Tenant { get; set; } - } -} diff --git a/src/WebJobs.Script.WebHost/Models/FunctionActivity.cs b/src/WebJobs.Script.WebHost/Models/FunctionActivity.cs deleted file mode 100644 index 50bee7cf9b..0000000000 --- a/src/WebJobs.Script.WebHost/Models/FunctionActivity.cs +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System; -using System.Runtime.Serialization; - -namespace Microsoft.Azure.WebJobs.Script.WebHost.Models -{ - public enum FunctionExecutionStage - { - /// - /// The function is currently executing. - /// - [EnumMember] - InProgress, - - /// - /// The function has finished executing. - /// - [EnumMember] - Finished - } - - public class FunctionActivity : ActivityBase - { - public string FunctionName { get; set; } - - public string InvocationId { get; set; } - - public int Concurrency { get; set; } - - public string ExecutionId { get; set; } - - public FunctionExecutionStage ExecutionStage { get; set; } - - public bool IsSucceeded { get; set; } - - public long ExecutionTimeSpanInMs { get; set; } - - public DateTime StartTime { get; set; } - } -} diff --git a/src/WebJobs.Script.WebHost/Models/MemoryActivity.cs b/src/WebJobs.Script.WebHost/Models/MemoryActivity.cs deleted file mode 100644 index c066494628..0000000000 --- a/src/WebJobs.Script.WebHost/Models/MemoryActivity.cs +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright (c) .NET Foundation. All rights reserved. -// Licensed under the MIT License. See License.txt in the project root for license information. - -using System; - -namespace Microsoft.Azure.WebJobs.Script.WebHost.Models -{ - public class MemoryActivity : ActivityBase - { - public long CommitSizeInBytes { get; set; } - } -} diff --git a/src/WebJobs.Script.WebHost/WebHostServiceCollectionExtensions.cs b/src/WebJobs.Script.WebHost/WebHostServiceCollectionExtensions.cs index 2d822492b8..61fb9f5940 100644 --- a/src/WebJobs.Script.WebHost/WebHostServiceCollectionExtensions.cs +++ b/src/WebJobs.Script.WebHost/WebHostServiceCollectionExtensions.cs @@ -5,6 +5,7 @@ using System.Net.Http; using System.Runtime.InteropServices; using Microsoft.AspNetCore.Authorization; +using Microsoft.Azure.Functions.Platform.Metrics.LinuxConsumption; using Microsoft.Azure.WebJobs.Extensions.Http; using Microsoft.Azure.WebJobs.Host.Storage; using Microsoft.Azure.WebJobs.Script.Config; @@ -256,6 +257,11 @@ private static void AddStandbyServices(this IServiceCollection services) private static void AddLinuxContainerServices(this IServiceCollection services) { + if (SystemEnvironment.Instance.IsV1LinuxConsumptionOnLegion()) + { + services.AddLinuxConsumptionMetricsServices(); + } + services.AddSingleton(s => { var environment = s.GetService(); @@ -280,7 +286,15 @@ private static void AddLinuxContainerServices(this IServiceCollection services) services.AddSingleton(s => { var environment = s.GetService(); - if (environment.IsFlexConsumptionSku()) + if (environment.IsV1LinuxConsumptionOnLegion()) + { + var logger = s.GetService>(); + var metricsTracker = s.GetService(); + var standbyOptions = s.GetService>(); + //var metricsLogger = s.GetService(); + return new LinuxContainerLegionMetricsPublisher(environment, standbyOptions, logger, new FileSystem(), metricsTracker); + } + else if (environment.IsFlexConsumptionSku()) { var options = s.GetService>(); var standbyOptions = s.GetService>(); diff --git a/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj b/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj index e28863a1f1..2d447f2d48 100644 --- a/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj +++ b/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj @@ -58,6 +58,7 @@ + diff --git a/src/WebJobs.Script/Environment/EnvironmentExtensions.cs b/src/WebJobs.Script/Environment/EnvironmentExtensions.cs index 91b8102a38..25197dd83b 100644 --- a/src/WebJobs.Script/Environment/EnvironmentExtensions.cs +++ b/src/WebJobs.Script/Environment/EnvironmentExtensions.cs @@ -335,6 +335,41 @@ private static bool IsLinuxConsumptionOnLegion(this IEnvironment environment) !string.IsNullOrEmpty(environment.GetEnvironmentVariable(LegionServiceHost)); } + /// + /// Returns a value indicating whether the app is V1 Linux Consumption running on Legion. + /// + /// The environment to verify. + /// if the app is V1 Linux Consumption running on Legion; otherwise, . + public static bool IsV1LinuxConsumptionOnLegion(this IEnvironment environment) + { + return IsLinuxConsumptionOnLegion(environment); //&& environment.WebsiteSkuIsDynamic(); + } + + /// + /// Checks both WEBSITE_SKU and WEBSITE_SKU_NAME and returns true IFF one is + /// set to "Dynamic". + /// + /// The environment to check. + /// if the sku is Dynamic; otherwise, . + public static bool WebsiteSkuIsDynamic(this IEnvironment environment) + { + string value = environment.GetEnvironmentVariable(AzureWebsiteSku); + if (string.Equals(value, ScriptConstants.DynamicSku, StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + // Linux Consumption v1 uses WEBSITE_SKU_NAME but is migrating to use WEBSTIE_SKU. + // So for now, we must check both. + value = environment.GetEnvironmentVariable(AzureWebsiteSkuName); + if (string.Equals(value, ScriptConstants.DynamicSku, StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + return false; + } + /// /// Gets a value indicating whether the application is running in a Linux App Service /// environment (Dedicated Linux). diff --git a/src/WebJobs.Script/Environment/EnvironmentSettingNames.cs b/src/WebJobs.Script/Environment/EnvironmentSettingNames.cs index 4108e1a673..9d1c542875 100644 --- a/src/WebJobs.Script/Environment/EnvironmentSettingNames.cs +++ b/src/WebJobs.Script/Environment/EnvironmentSettingNames.cs @@ -11,6 +11,7 @@ public static class EnvironmentSettingNames public const string AzureWebsiteOwnerName = "WEBSITE_OWNER_NAME"; public const string AzureWebsiteInstanceId = "WEBSITE_INSTANCE_ID"; public const string AzureWebsiteSku = "WEBSITE_SKU"; + public const string AzureWebsiteSkuName = "WEBSITE_SKU_NAME"; public const string RemoteDebuggingPort = "REMOTEDEBUGGINGPORT"; public const string AzureWebsitePlaceholderMode = "WEBSITE_PLACEHOLDER_MODE"; public const string AzureWebsiteUsePlaceholderDotNetIsolated = "WEBSITE_USE_PLACEHOLDER_DOTNETISOLATED"; diff --git a/test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/MetricsEndToEndTests_FlexConsumption.cs b/test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/MetricsEndToEndTests_FlexConsumption.cs index 214134296c..0fb43971cd 100644 --- a/test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/MetricsEndToEndTests_FlexConsumption.cs +++ b/test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/MetricsEndToEndTests_FlexConsumption.cs @@ -35,7 +35,7 @@ public MetricsEndToEndTests_FlexConsumption(TestFixture fixture) // reset values that the tests are configuring var metricsPublisher = (FlexConsumptionMetricsPublisher)_fixture.Host.JobHostServices.GetService(); metricsPublisher.IsAlwaysReady = false; - metricsPublisher.MetricsFilePath = _fixture.MetricsPublishPath; + metricsPublisher.MetricsFileManager.MetricsFilePath = _fixture.MetricsPublishPath; _fixture.CleanupMetricsFiles(); } @@ -51,7 +51,7 @@ public async Task ShortTestRun_ExpectedMetricsGenerated(bool isAlwaysReadyInstan if (!metricsPublishEnabled) { - metricsPublisher.MetricsFilePath = null; + metricsPublisher.MetricsFileManager.MetricsFilePath = null; } int activityDuration = 2500; diff --git a/test/WebJobs.Script.Tests.Shared/TestTraits.cs b/test/WebJobs.Script.Tests.Shared/TestTraits.cs index 547576eb53..ec9cdba763 100644 --- a/test/WebJobs.Script.Tests.Shared/TestTraits.cs +++ b/test/WebJobs.Script.Tests.Shared/TestTraits.cs @@ -58,6 +58,8 @@ public static class TestTraits public const string FlexConsumptionMetricsTests = "FlexConsumptionMetricsTests"; + public const string LinuxConsumptionMetricsTests = "LinuxConsumptionMetricsTests"; + public const string HostMetricsTests = "HostMetricsTests"; public const string HISSecretsTests = "HISSecretsTests"; diff --git a/test/WebJobs.Script.Tests/Extensions/EnvironmentExtensionsTests.cs b/test/WebJobs.Script.Tests/Extensions/EnvironmentExtensionsTests.cs index f2f1eb35d4..f836035dbf 100644 --- a/test/WebJobs.Script.Tests/Extensions/EnvironmentExtensionsTests.cs +++ b/test/WebJobs.Script.Tests/Extensions/EnvironmentExtensionsTests.cs @@ -273,6 +273,33 @@ public void IsManagedAppEnvironment_ReturnsExpectedResult(bool isManagedAppEnvir Assert.Equal(expectedValue, env.IsManagedAppEnvironment()); } + [Theory] + [InlineData(false, null, null, false)] + [InlineData(false, "", "", false)] + [InlineData(false, ScriptConstants.DynamicSku, null, false)] + [InlineData(false, null, ScriptConstants.DynamicSku, false)] + [InlineData(true, null, null, false)] + [InlineData(true, "", "", false)] + [InlineData(true, ScriptConstants.FlexConsumptionSku, null, false)] + [InlineData(true, null, ScriptConstants.DynamicSku, true)] + [InlineData(true, ScriptConstants.DynamicSku, null, true)] + [Trait(TestTraits.Group, TestTraits.LinuxConsumptionMetricsTests)] + public void IsV1LinuxConsumptionOnLegion_ReturnsExpectedResult(bool isLinuxConsumptionOnLegion, string websiteSku, string websiteSkuName, bool expectedValue) + { + IEnvironment env = new TestEnvironment(); + + if (isLinuxConsumptionOnLegion) + { + env.SetEnvironmentVariable(WebsitePodName, "RandomPodName"); + env.SetEnvironmentVariable(LegionServiceHost, "RandomLegionServiceHostName"); + } + + env.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteSku, websiteSku); + env.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteSkuName, websiteSkuName); + + Assert.Equal(expectedValue, env.IsV1LinuxConsumptionOnLegion()); + } + [Theory] [InlineData("~2", "true", true)] [InlineData("~2", "false", true)] diff --git a/test/WebJobs.Script.Tests/Metrics/LinuxContainerLegionMetricsPublisherTests.cs b/test/WebJobs.Script.Tests/Metrics/LinuxContainerLegionMetricsPublisherTests.cs new file mode 100644 index 0000000000..f4afd750c9 --- /dev/null +++ b/test/WebJobs.Script.Tests/Metrics/LinuxContainerLegionMetricsPublisherTests.cs @@ -0,0 +1,258 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Abstractions; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Azure.Functions.Platform.Metrics.LinuxConsumption; +using Microsoft.Azure.WebJobs.Script.WebHost; +using Microsoft.Azure.WebJobs.Script.WebHost.Metrics; +using Microsoft.WebJobs.Script.Tests; +using Newtonsoft.Json; +using Xunit; + +namespace Microsoft.Azure.WebJobs.Script.Tests.Metrics +{ + [Trait(TestTraits.Group, TestTraits.LinuxConsumptionMetricsTests)] + public class LinuxContainerLegionMetricsPublisherTests + { + private const string TestFunctionName = "testfunction"; + + private readonly string _metricsFilePath; + private readonly IEnvironment _environment; + private readonly TestMetricsTracker _testMetricsTracker; + private readonly Random _random = new Random(); + private readonly TestLogger _logger; + private readonly TestMetricsLogger _testMetricsLogger; + + private StandbyOptions _standbyOptions; + private TestOptionsMonitor _standbyOptionsMonitor; + + public LinuxContainerLegionMetricsPublisherTests() + { + _metricsFilePath = Path.Combine(Path.GetTempPath(), "metrics"); + _environment = new TestEnvironment(); + _logger = new TestLogger(); + _testMetricsLogger = new TestMetricsLogger(); + + _environment.SetEnvironmentVariable(EnvironmentSettingNames.FunctionsMetricsPublishPath, _metricsFilePath); + + CleanupMetricsFiles(); + _testMetricsTracker = new TestMetricsTracker(); + } + + private LinuxContainerLegionMetricsPublisher CreatePublisher(bool inStandbyMode = false, int? metricsPublishInterval = null) + { + _standbyOptions = new StandbyOptions { InStandbyMode = inStandbyMode }; + _standbyOptionsMonitor = new TestOptionsMonitor(_standbyOptions); + + return new LinuxContainerLegionMetricsPublisher(_environment, _standbyOptionsMonitor, _logger, new FileSystem(), _testMetricsLogger, _testMetricsTracker, metricsPublishInterval); + } + + [Fact] + public async Task AddActivities_ExpectedActivitiesArePublished() + { + var metricsPublisher = CreatePublisher(inStandbyMode: true); + metricsPublisher.Initialize(); + + await AddTestActivities(metricsPublisher, 10, 25); + + Assert.Equal(10, _testMetricsTracker.FunctionActivities.Count); + var functionActivity = _testMetricsTracker.FunctionActivities[0]; + Assert.Equal(TestFunctionName, functionActivity.FunctionName); + Assert.Equal(FunctionExecutionStage.Finished, functionActivity.ExecutionStage); + + Assert.Equal(25, _testMetricsTracker.MemoryActivities.Count); + var memoryActivity = _testMetricsTracker.MemoryActivities[0]; + } + + [Fact] + public async Task AddActivities_StandbyMode_ActivitiesNotPublished() + { + var metricsPublisher = CreatePublisher(inStandbyMode: true); + + await AddTestActivities(metricsPublisher, 10, 25); + + Assert.Equal(0, _testMetricsTracker.FunctionActivities.Count); + Assert.Equal(0, _testMetricsTracker.MemoryActivities.Count); + } + + [Fact] + public async Task PublishMetrics_WritesExpectedFile() + { + var newMetrics = new LinuxConsumptionMetrics + { + FunctionExecutionCount = 111, + FunctionExecutionTimeMS = 222, + FunctionActivity = 333333 + }; + _testMetricsTracker.MetricsQueue.Enqueue(newMetrics); + + var metricsPublisher = CreatePublisher(inStandbyMode: true); + + await metricsPublisher.OnPublishMetrics(); + + FileInfo[] metricsFiles = GetMetricsFilesSafe(_metricsFilePath); + Assert.Equal(1, metricsFiles.Length); + + var metrics = await ReadMetricsAsync(metricsFiles[0].FullName); + + Assert.Equal(newMetrics.FunctionExecutionCount, metrics.ExecutionCount); + Assert.Equal(newMetrics.FunctionExecutionTimeMS, metrics.ExecutionTimeMS); + Assert.Equal(newMetrics.FunctionActivity, metrics.FunctionActivity); + } + + [Fact] + public async Task MetricsFilesPublishedOnInterval() + { + EnqueueTestMetrics(5); + + var metricsPublisher = CreatePublisher(metricsPublishInterval: 100); + + FileInfo[] metricsFiles = null; + await TestHelpers.Await(() => + { + metricsFiles = GetMetricsFilesSafe(_metricsFilePath); + return metricsFiles.Length == 5; + }); + + Assert.Equal(5, metricsFiles.Length); + } + + [Fact] + public async Task MetricsFilesNotPublished_WhenMetricsNotAvailable() + { + var metricsPublisher = CreatePublisher(metricsPublishInterval: 100); + + await Task.Delay(500); + + FileInfo[] metricsFiles = GetMetricsFilesSafe(_metricsFilePath); + Assert.Empty(metricsFiles); + } + + [Fact] + public void LogEvent_LogsMetricEvent() + { + var metricsPublisher = CreatePublisher(); + + for (int i = 0; i < 10; i++) + { + _testMetricsTracker.LogEvent("testevent1"); + } + + for (int i = 0; i < 5; i++) + { + _testMetricsTracker.LogEvent("testevent2"); + } + + Assert.Equal(15, _testMetricsLogger.LoggedEvents.Count); + Assert.Equal(10, _testMetricsLogger.LoggedEvents.Count(p => p == "testevent1")); + Assert.Equal(5, _testMetricsLogger.LoggedEvents.Count(p => p == "testevent2")); + } + + private void EnqueueTestMetrics(int numMetrics) + { + for (int i = 0; i < numMetrics; i++) + { + _testMetricsTracker.MetricsQueue.Enqueue(new LinuxConsumptionMetrics + { + FunctionExecutionCount = _random.Next(1, 100), + FunctionExecutionTimeMS = _random.Next(1, 100), + FunctionActivity = _random.Next(10000, 1000000) + }); + } + } + + private async Task AddTestActivities(LinuxContainerLegionMetricsPublisher metricsPublisher, int numFunctionActivities, int numMemoryActivities) + { + var t1 = Task.Run(async () => + { + for (int i = 0; i < numFunctionActivities; i++) + { + var startTime = DateTime.UtcNow; + await Task.Delay(25); + var endTime = DateTime.UtcNow; + var duration = endTime - startTime; + metricsPublisher.AddFunctionExecutionActivity(TestFunctionName, Guid.NewGuid().ToString(), 50, FunctionExecutionStage.Finished.ToString(), true, (long)duration.TotalMilliseconds, Guid.NewGuid().ToString(), DateTime.UtcNow, startTime); + } + }); + + var t2 = Task.Run(async () => + { + for (int i = 0; i < numMemoryActivities; i++) + { + await Task.Delay(10); + metricsPublisher.AddMemoryActivity(DateTime.UtcNow, 1000); + } + }); + + await Task.WhenAll(t1, t2); + } + + public void CleanupMetricsFiles() + { + var directory = new DirectoryInfo(_metricsFilePath); + + if (!directory.Exists) + { + return; + } + + foreach (var file in directory.GetFiles()) + { + file.Delete(); + } + } + + private static async Task ReadMetricsAsync(string metricsFilePath) + { + string content = await File.ReadAllTextAsync(metricsFilePath); + return JsonConvert.DeserializeObject(content); + } + + private static FileInfo[] GetMetricsFilesSafe(string path) + { + var directory = new DirectoryInfo(path); + if (directory.Exists) + { + return directory.GetFiles().OrderBy(p => p.CreationTime).ToArray(); + } + + return new FileInfo[0]; + } + + private class TestMetricsTracker : ILinuxConsumptionMetricsTracker + { + public event EventHandler OnDiagnosticEvent; + + public List FunctionActivities { get; } = new List(); + + public List MemoryActivities { get; } = new List(); + + public Queue MetricsQueue { get; } = new Queue(); + + public void AddFunctionActivity(FunctionActivity activity) + { + FunctionActivities.Add(activity); + } + + public void AddMemoryActivity(MemoryActivity activity) + { + MemoryActivities.Add(activity); + } + + public bool TryGetMetrics(out LinuxConsumptionMetrics metrics) + { + return MetricsQueue.TryDequeue(out metrics); + } + + public void LogEvent(string eventName) + { + OnDiagnosticEvent?.Invoke(this, new DiagnosticEventArgs(eventName)); + } + } + } +} \ No newline at end of file diff --git a/test/WebJobs.Script.Tests/Metrics/LinuxContainerMetricsPublisherTests.cs b/test/WebJobs.Script.Tests/Metrics/LinuxContainerMetricsPublisherTests.cs index 0fcd0c3a34..db7647113c 100644 --- a/test/WebJobs.Script.Tests/Metrics/LinuxContainerMetricsPublisherTests.cs +++ b/test/WebJobs.Script.Tests/Metrics/LinuxContainerMetricsPublisherTests.cs @@ -9,10 +9,10 @@ using System.Net.Http; using System.Threading; using System.Threading.Tasks; +using Microsoft.Azure.Functions.Platform.Metrics.LinuxConsumption; using Microsoft.Azure.WebJobs.Script.Config; using Microsoft.Azure.WebJobs.Script.WebHost; using Microsoft.Azure.WebJobs.Script.WebHost.Metrics; -using Microsoft.Azure.WebJobs.Script.WebHost.Models; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Microsoft.WebJobs.Script.Tests; @@ -22,6 +22,7 @@ namespace Microsoft.Azure.WebJobs.Script.Tests.Metrics { + [Trait(TestTraits.Group, TestTraits.LinuxConsumptionMetricsTests)] public class LinuxContainerMetricsPublisherTests { private const string _containerName = "test-container";