Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

User/vijilla/legion cv1 metrics #10750

Draft
wants to merge 3 commits into
base: dev
Choose a base branch
from
Draft

User/vijilla/legion cv1 metrics #10750

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ef86f05
Linux Consumption metrics publisher for Legion
vivekjilla Jan 16, 2025
d8c9b85
nit
vivekjilla Jan 16, 2025
20d43c1
Removing cyclic dependency between metricslogger and metricspublisher
vivekjilla Jan 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions NuGet.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,6 @@
<add key="AzureFunctionsRelease" value="https://azfunc.pkgs.visualstudio.com/e6a70c92-4128-439f-8012-382fe78d6396/_packaging/AzureFunctionsRelease/nuget/v3/index.json" />
<add key="AzureFunctionsPreRelease" value="https://azfunc.pkgs.visualstudio.com/e6a70c92-4128-439f-8012-382fe78d6396/_packaging/AzureFunctionsPreRelease/nuget/v3/index.json" />
<add key="AzureFunctionsTempStaging" value="https://azfunc.pkgs.visualstudio.com/e6a70c92-4128-439f-8012-382fe78d6396/_packaging/AzureFunctionsTempStaging/nuget/v3/index.json" />
<add key="AzureFunctions" value="https://pkgs.dev.azure.com/azfunc/e6a70c92-4128-439f-8012-382fe78d6396/_packaging/AzureFunctions/nuget/v3/index.json" />
</packageSources>
</configuration>
92 changes: 10 additions & 82 deletions src/WebJobs.Script.WebHost/Metrics/FlexConsumptionMetricsPublisher.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ public class FlexConsumptionMetricsPublisher : IMetricsPublisher, IDisposable
private readonly IHostMetricsProvider _metricsProvider;
private readonly object _lock = new object();
private readonly IFileSystem _fileSystem;
private readonly LegionMetricsFileManager _metricsFileManager;

private Timer _metricsPublisherTimer;
private bool _started = false;
Expand All @@ -44,6 +45,7 @@ public FlexConsumptionMetricsPublisher(IEnvironment environment, IOptionsMonitor
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_fileSystem = fileSystem ?? new FileSystem();
_metricsFileManager = new LegionMetricsFileManager(_options.MetricsFilePath, _fileSystem, _logger, _options.MaxFileCount);
_metricsProvider = metricsProvider ?? throw new ArgumentNullException(nameof(metricsProvider));

if (_standbyOptions.CurrentValue.InStandbyMode)
Expand All @@ -66,13 +68,13 @@ public FlexConsumptionMetricsPublisher(IEnvironment environment, IOptionsMonitor

internal bool IsAlwaysReady { get; set; }

internal string MetricsFilePath { get; set; }
internal LegionMetricsFileManager MetricsFileManager => _metricsFileManager;

public void Start()
{
Initialize();

_logger.LogInformation($"Starting metrics publisher (AlwaysReady={IsAlwaysReady}, MetricsPath='{MetricsFilePath}').");
_logger.LogInformation($"Starting metrics publisher (AlwaysReady={IsAlwaysReady}, MetricsPath='{_metricsFileManager.MetricsFilePath}').");

_metricsPublisherTimer = new Timer(OnFunctionMetricsPublishTimer, null, _initialPublishDelay, _metricPublishInterval);
_started = true;
Expand All @@ -86,7 +88,6 @@ internal void Initialize()
_metricPublishInterval = TimeSpan.FromMilliseconds(_options.MetricsPublishIntervalMS);
_initialPublishDelay = TimeSpan.FromMilliseconds(_options.InitialPublishDelayMS);
_intervalStopwatch = ValueStopwatch.StartNew();
MetricsFilePath = _options.MetricsFilePath;

IsAlwaysReady = _environment.GetEnvironmentVariable(EnvironmentSettingNames.FunctionsAlwaysReadyInstance) == "1";
}
Expand Down Expand Up @@ -136,7 +137,12 @@ internal async Task OnPublishMetrics(DateTime now)
FunctionExecutionTimeMS = FunctionExecutionCount = 0;
}

await PublishMetricsAsync(metrics);
await _metricsFileManager.PublishMetricsAsync(metrics);
}
catch (Exception ex) when (!ex.IsFatal())
{
// ensure no background exceptions escape
_logger.LogError(ex, $"Error publishing metrics.");
}
finally
{
Expand All @@ -149,84 +155,6 @@ private async void OnFunctionMetricsPublishTimer(object state)
await OnPublishMetrics(DateTime.UtcNow);
}

private async Task PublishMetricsAsync(Metrics metrics)
{
string fileName = string.Empty;

try
{
bool metricsPublishEnabled = !string.IsNullOrEmpty(MetricsFilePath);
if (metricsPublishEnabled && !PrepareDirectoryForFile())
{
return;
}

string metricsContent = JsonConvert.SerializeObject(metrics);
_logger.PublishingMetrics(metricsContent);

if (metricsPublishEnabled)
{
fileName = $"{Guid.NewGuid().ToString().ToLower()}.json";
string filePath = Path.Combine(MetricsFilePath, fileName);

using (var streamWriter = _fileSystem.File.CreateText(filePath))
{
await streamWriter.WriteAsync(metricsContent);
}
}
}
catch (Exception ex) when (!ex.IsFatal())
{
// TODO: consider using a retry strategy here
_logger.LogError(ex, $"Error writing metrics file '{fileName}'.");
}
}

private bool PrepareDirectoryForFile()
{
if (string.IsNullOrEmpty(MetricsFilePath))
{
return false;
}

// ensure the directory exists
_fileSystem.Directory.CreateDirectory(MetricsFilePath);

var metricsDirectoryInfo = _fileSystem.DirectoryInfo.FromDirectoryName(MetricsFilePath);
var files = metricsDirectoryInfo.GetFiles().OrderBy(p => p.CreationTime).ToList();

// ensure we're under the max file count
if (files.Count < _options.MaxFileCount)
{
return true;
}

// we're at or over limit
// delete enough files that we have space to write a new one
int numToDelete = files.Count - _options.MaxFileCount + 1;
var filesToDelete = files.Take(numToDelete).ToArray();

_logger.LogDebug($"Deleting {filesToDelete.Length} metrics file(s).");

foreach (var file in filesToDelete)
{
try
{
file.Delete();
}
catch (Exception ex) when (!ex.IsFatal())
{
// best effort
_logger.LogError(ex, $"Error deleting metrics file '{file.FullName}'.");
}
}

files = metricsDirectoryInfo.GetFiles().OrderBy(p => p.CreationTime).ToList();

// return true if we have space for a new file
return files.Count < _options.MaxFileCount;
}

private void OnStandbyOptionsChange()
{
if (!_standbyOptions.CurrentValue.InStandbyMode)
Expand Down
109 changes: 109 additions & 0 deletions src/WebJobs.Script.WebHost/Metrics/LegionMetricsFileManager.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the MIT License. See License.txt in the project root for license information.

using System;
using System.IO;
using System.IO.Abstractions;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.Azure.WebJobs.Script.Diagnostics.Extensions;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;

namespace Microsoft.Azure.WebJobs.Script.WebHost.Metrics
{
public class LegionMetricsFileManager
{
private readonly IFileSystem _fileSystem;
private readonly int _maxFileCount;
private readonly ILogger _logger;

public LegionMetricsFileManager(string metricsFilePath, IFileSystem fileSystem, ILogger logger, int maxFileCount)
{
_fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
MetricsFilePath = metricsFilePath;
_maxFileCount = maxFileCount;
}

internal string MetricsFilePath { get; set; }

private bool PrepareDirectoryForFile()
{
if (string.IsNullOrEmpty(MetricsFilePath))
{
return false;
}

// ensure the directory exists
_fileSystem.Directory.CreateDirectory(MetricsFilePath);

var metricsDirectoryInfo = _fileSystem.DirectoryInfo.FromDirectoryName(MetricsFilePath);
var files = metricsDirectoryInfo.GetFiles().OrderBy(p => p.CreationTime).ToList();

// ensure we're under the max file count
if (files.Count < _maxFileCount)
{
return true;
}

// we're at or over limit
// delete enough files that we have space to write a new one
int numToDelete = files.Count - _maxFileCount + 1;
var filesToDelete = files.Take(numToDelete).ToArray();

_logger.LogDebug($"Deleting {filesToDelete.Length} metrics file(s).");

foreach (var file in filesToDelete)
{
try
{
file.Delete();
}
catch (Exception ex) when (!ex.IsFatal())
{
// best effort
_logger.LogError(ex, $"Error deleting metrics file '{file.FullName}'.");
}
}

files = metricsDirectoryInfo.GetFiles().OrderBy(p => p.CreationTime).ToList();

// return true if we have space for a new file
return files.Count < _maxFileCount;
}

public async Task PublishMetricsAsync(object metrics)
{
string fileName = string.Empty;

try
{
bool metricsPublishEnabled = !string.IsNullOrEmpty(MetricsFilePath);
if (metricsPublishEnabled && !PrepareDirectoryForFile())
{
return;
}

string metricsContent = JsonConvert.SerializeObject(metrics);
_logger.PublishingMetrics(metricsContent);

if (metricsPublishEnabled)
{
fileName = $"{Guid.NewGuid().ToString().ToLower()}.json";
string filePath = Path.Combine(MetricsFilePath, fileName);

using (var streamWriter = _fileSystem.File.CreateText(filePath))
{
await streamWriter.WriteAsync(metricsContent);
}
}
}
catch (Exception ex) when (!ex.IsFatal())
{
// TODO: consider using a retry strategy here
_logger.LogError(ex, $"Error writing metrics file '{fileName}'.");
}
}
}
}
Loading
Loading