Skip to content

Commit 45d0e5c

Browse files
mathewcmathewc
committed
HostID Collision detection
1 parent b2c9572 commit 45d0e5c

File tree

11 files changed

+552
-31
lines changed

11 files changed

+552
-31
lines changed

‎src/WebJobs.Script.WebHost/WebHostServiceCollectionExtensions.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,6 @@ public static void AddWebJobsScriptHost(this IServiceCollection services, IConfi
124124
services.AddSingleton<IInstanceManager, InstanceManager>();
125125
services.AddSingleton(_ => new HttpClient());
126126
services.AddSingleton<StartupContextProvider>();
127-
services.AddSingleton<HostNameProvider>();
128127
services.AddSingleton<IFileSystem>(_ => FileUtility.Instance);
129128
services.AddTransient<VirtualFileSystem>();
130129
services.AddTransient<VirtualFileSystemMiddleware>();

‎src/WebJobs.Script/Environment/EnvironmentSettingNames.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ public static class EnvironmentSettingNames
5858
public const string TestDataCapEnabled = "WEBSITE_FUNCTIONS_TESTDATA_CAP_ENABLED";
5959
public const string AzureMonitorCategories = "WEBSITE_FUNCTIONS_AZUREMONITOR_CATEGORIES";
6060
public const string FunctionsRequestBodySizeLimit = "FUNCTIONS_REQUEST_BODY_SIZE_LIMIT";
61+
public const string FunctionsHostIdCheckLevel = "FUNCTIONS_HOSTID_CHECK_LEVEL";
6162

6263
//Function in Kubernetes
6364
public const string PodNamespace = "POD_NAMESPACE";

‎src/WebJobs.Script/Host/HostIdValidator.cs

Lines changed: 213 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,213 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the MIT License. See License.txt in the project root for license information.
3+
4+
using System;
5+
using System.IO;
6+
using System.Threading.Tasks;
7+
using Azure;
8+
using Azure.Storage.Blobs;
9+
using Microsoft.Azure.WebJobs.Hosting;
10+
using Microsoft.Azure.WebJobs.Script.Properties;
11+
using Microsoft.Extensions.DependencyInjection;
12+
using Microsoft.Extensions.Logging;
13+
using Newtonsoft.Json;
14+
using IApplicationLifetime = Microsoft.AspNetCore.Hosting.IApplicationLifetime;
15+
16+
namespace Microsoft.Azure.WebJobs.Script
17+
{
18+
/// <summary>
19+
/// Used to perform Host ID validation checks, ensuring that when hosts are sharing
20+
/// a storage account, their computed IDs don't collide.
21+
/// </summary>
22+
/// <remarks>
23+
/// <see cref="ScriptHostIdProvider"/> computes a Host ID and truncates it if needed to
24+
/// ensure it's under length limits. For two different Function Apps, this can result in
25+
/// both apps resolving to the same Host ID. This can cause problems if those apps share
26+
/// a storage account. This class helps detect/prevent such cases.
27+
/// </remarks>
28+
public class HostIdValidator
29+
{
30+
public const string BlobPathFormat = "ids/usage/{0}";
31+
private const LogLevel DefaultLevel = LogLevel.Warning;
32+
33+
private readonly IEnvironment _environment;
34+
private readonly IAzureStorageProvider _storageProvider;
35+
private readonly IServiceProvider _serviceProvider;
36+
private readonly IApplicationLifetime _applicationLifetime;
37+
private readonly HostNameProvider _hostNameProvider;
38+
private readonly ILogger _logger;
39+
40+
private readonly object _syncLock = new object();
41+
private bool _validationScheduled;
42+
43+
public HostIdValidator(IEnvironment environment, IAzureStorageProvider storageProvider, IApplicationLifetime applicationLifetime,
44+
HostNameProvider hostNameProvider, ILogger<HostIdValidator> logger, IServiceProvider serviceProvider)
45+
{
46+
_environment = environment;
47+
_storageProvider = storageProvider;
48+
_serviceProvider = serviceProvider;
49+
_applicationLifetime = applicationLifetime;
50+
_hostNameProvider = hostNameProvider;
51+
_logger = logger;
52+
}
53+
54+
internal bool ValidationScheduled => _validationScheduled;
55+
56+
public virtual void ScheduleValidation(string hostId, int validationDelaySeconds = 10)
57+
{
58+
lock (_syncLock)
59+
{
60+
if (!_validationScheduled)
61+
{
62+
// Schedule the validation to run asynchronously after a delay. This delay ensures
63+
// we're not impacting coldstart, and also gives time for the primary host to be
64+
// identified.
65+
Utility.ExecuteAfterDelay(() => Task.Run(() => ValidateHostIdUsageAsync(hostId)), TimeSpan.FromSeconds(validationDelaySeconds));
66+
_validationScheduled = true;
67+
}
68+
}
69+
}
70+
71+
internal async Task ValidateHostIdUsageAsync(string hostId)
72+
{
73+
try
74+
{
75+
if (!_storageProvider.ConnectionExists(ConnectionStringNames.Storage))
76+
{
77+
return;
78+
}
79+
80+
var primaryHostStateProvider = _serviceProvider.GetScriptHostServiceOrNull<IPrimaryHostStateProvider>();
81+
if (primaryHostStateProvider != null && !primaryHostStateProvider.IsPrimary)
82+
{
83+
// as an optimization, only validate on the primary host
84+
return;
85+
}
86+
87+
HostIdInfo hostIdInfo = await ReadHostIdInfoAsync(hostId);
88+
89+
if (hostIdInfo != null)
90+
{
91+
// an existing record exists for this host ID
92+
CheckForCollision(hostId, hostIdInfo);
93+
}
94+
else
95+
{
96+
// no existing record, so write one, claiming this host ID for this host name
97+
// in this storage account
98+
hostIdInfo = new HostIdInfo
99+
{
100+
Hostname = _hostNameProvider.Value
101+
};
102+
await WriteHostIdAsync(hostId, hostIdInfo);
103+
}
104+
}
105+
catch (Exception ex)
106+
{
107+
// best effort - log error and continue
108+
_logger.LogError(ex, "Error validating host ID usage.");
109+
}
110+
}
111+
112+
private void CheckForCollision(string hostId, HostIdInfo hostIdInfo)
113+
{
114+
// verify the host name is the same as our host name
115+
if (!string.Equals(_hostNameProvider.Value, hostIdInfo.Hostname, StringComparison.OrdinalIgnoreCase))
116+
{
117+
HandleCollision(hostId);
118+
}
119+
}
120+
121+
private void HandleCollision(string hostId)
122+
{
123+
// see if the user has specified a level, otherwise default
124+
string value = _environment.GetEnvironmentVariable(EnvironmentSettingNames.FunctionsHostIdCheckLevel);
125+
if (!Enum.TryParse<LogLevel>(value, out LogLevel level))
126+
{
127+
level = DefaultLevel;
128+
}
129+
130+
string message = string.Format(Resources.HostIdCollisionFormat, hostId);
131+
if (level == LogLevel.Error)
132+
{
133+
_logger.LogError(message);
134+
_applicationLifetime.StopApplication();
135+
}
136+
else
137+
{
138+
// we only allow Warning/Error levels to be specified, so anything other than
139+
// Error is treated as warning
140+
_logger.LogWarning(message);
141+
}
142+
}
143+
144+
internal async Task WriteHostIdAsync(string hostId, HostIdInfo hostIdInfo)
145+
{
146+
try
147+
{
148+
var containerClient = _storageProvider.GetBlobContainerClient();
149+
string blobPath = string.Format(BlobPathFormat, hostId);
150+
BlobClient blobClient = containerClient.GetBlobClient(blobPath);
151+
BinaryData data = BinaryData.FromObjectAsJson(hostIdInfo);
152+
await blobClient.UploadAsync(data);
153+
154+
_logger.LogDebug($"Host ID record written (ID:{hostId}, HostName:{hostIdInfo.Hostname})");
155+
}
156+
catch (RequestFailedException rfex) when (rfex.Status == 409)
157+
{
158+
// Another instance wrote the blob between the time when we initially
159+
// checked and when we attempted to write. Read the blob and validate it.
160+
hostIdInfo = await ReadHostIdInfoAsync(hostId);
161+
if (hostIdInfo != null)
162+
{
163+
CheckForCollision(hostId, hostIdInfo);
164+
}
165+
}
166+
catch (Exception ex)
167+
{
168+
// best effort
169+
_logger.LogError(ex, "Error writing host ID info");
170+
}
171+
}
172+
173+
internal async Task<HostIdInfo> ReadHostIdInfoAsync(string hostId)
174+
{
175+
HostIdInfo hostIdInfo = null;
176+
177+
try
178+
{
179+
// check storage to see if a record already exists for this host ID
180+
var containerClient = _storageProvider.GetBlobContainerClient();
181+
string blobPath = string.Format(BlobPathFormat, hostId);
182+
BlobClient blobClient = containerClient.GetBlobClient(blobPath);
183+
var downloadResponse = await blobClient.DownloadAsync();
184+
string content;
185+
using (StreamReader reader = new StreamReader(downloadResponse.Value.Content))
186+
{
187+
content = reader.ReadToEnd();
188+
}
189+
190+
if (!string.IsNullOrEmpty(content))
191+
{
192+
hostIdInfo = JsonConvert.DeserializeObject<HostIdInfo>(content);
193+
}
194+
}
195+
catch (RequestFailedException exception) when (exception.Status == 404)
196+
{
197+
// no record stored for this host ID
198+
}
199+
catch (Exception ex)
200+
{
201+
// best effort
202+
_logger.LogError(ex, "Error reading host ID info");
203+
}
204+
205+
return hostIdInfo;
206+
}
207+
208+
internal class HostIdInfo
209+
{
210+
public string Hostname { get; set; }
211+
}
212+
}
213+
}

‎src/WebJobs.Script/Host/ScriptHost.cs

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,6 @@ public class ScriptHost : JobHost, IScriptJobHost
4646
internal const string GeneratedTypeName = "Functions";
4747
private readonly IApplicationLifetime _applicationLifetime;
4848
private readonly IScriptHostManager _scriptHostManager;
49-
private readonly string _storageConnectionString;
5049
private readonly IDistributedLockManager _distributedLockManager;
5150
private readonly IFunctionMetadataManager _functionMetadataManager;
5251
private readonly IFileLoggingStatusManager _fileLoggingStatusManager;
@@ -118,7 +117,6 @@ public ScriptHost(IOptions<JobHostOptions> options,
118117
_instanceId = Guid.NewGuid().ToString();
119118
_hostOptions = options;
120119
_configuration = configuration;
121-
_storageConnectionString = configuration.GetWebJobsConnectionString(ConnectionStringNames.Storage);
122120
_distributedLockManager = distributedLockManager;
123121
_functionMetadataManager = functionMetadataManager;
124122
_fileLoggingStatusManager = fileLoggingStatusManager;

‎src/WebJobs.Script/Host/ScriptHostIdProvider.cs

Lines changed: 33 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,21 +18,36 @@ public class ScriptHostIdProvider : IHostIdProvider
1818
private readonly IConfiguration _config;
1919
private readonly IEnvironment _environment;
2020
private readonly IOptionsMonitor<ScriptApplicationHostOptions> _options;
21+
private readonly HostIdValidator _hostIdValidator;
2122

22-
public ScriptHostIdProvider(IConfiguration config, IEnvironment environment, IOptionsMonitor<ScriptApplicationHostOptions> options)
23+
public ScriptHostIdProvider(IConfiguration config, IEnvironment environment, IOptionsMonitor<ScriptApplicationHostOptions> options, HostIdValidator hostIdValidator)
2324
{
2425
_config = config;
2526
_environment = environment;
2627
_options = options;
28+
_hostIdValidator = hostIdValidator;
2729
}
2830

2931
public Task<string> GetHostIdAsync(CancellationToken cancellationToken)
3032
{
31-
return Task.FromResult(_config[ConfigurationSectionNames.HostIdPath] ?? GetDefaultHostId(_environment, _options.CurrentValue));
33+
string hostId = _config[ConfigurationSectionNames.HostIdPath];
34+
if (hostId == null)
35+
{
36+
HostIdResult result = GetDefaultHostId(_environment, _options.CurrentValue);
37+
hostId = result.HostId;
38+
if (result.IsTruncated && !result.IsLocal)
39+
{
40+
_hostIdValidator.ScheduleValidation(hostId);
41+
}
42+
}
43+
44+
return Task.FromResult(hostId);
3245
}
3346

34-
internal static string GetDefaultHostId(IEnvironment environment, ScriptApplicationHostOptions scriptOptions)
47+
internal static HostIdResult GetDefaultHostId(IEnvironment environment, ScriptApplicationHostOptions scriptOptions)
3548
{
49+
HostIdResult result = new HostIdResult();
50+
3651
// We're setting the default here on the newly created configuration
3752
// If the user has explicitly set the HostID via host.json, it will overwrite
3853
// what we set here
@@ -64,19 +79,32 @@ internal static string GetDefaultHostId(IEnvironment environment, ScriptApplicat
6479
.Where(char.IsLetterOrDigit)
6580
.Aggregate(new StringBuilder(), (b, c) => b.Append(c)).ToString();
6681
hostId = $"{sanitizedMachineName}-{Math.Abs(Utility.GetStableHash(scriptOptions.ScriptPath))}";
82+
result.IsLocal = true;
6783
}
6884

6985
if (!string.IsNullOrEmpty(hostId))
7086
{
7187
if (hostId.Length > ScriptConstants.MaximumHostIdLength)
7288
{
73-
// Truncate to the max host name length if needed
89+
// Truncate to the max host name length
7490
hostId = hostId.Substring(0, ScriptConstants.MaximumHostIdLength);
91+
result.IsTruncated = true;
7592
}
7693
}
7794

7895
// Lowercase and trim any trailing '-' as they can cause problems with queue names
79-
return hostId?.ToLowerInvariant().TrimEnd('-');
96+
result.HostId = hostId?.ToLowerInvariant().TrimEnd('-');
97+
98+
return result;
99+
}
100+
101+
public class HostIdResult
102+
{
103+
public string HostId { get; set; }
104+
105+
public bool IsTruncated { get; set; }
106+
107+
public bool IsLocal { get; set; }
80108
}
81109
}
82110
}

‎src/WebJobs.Script.WebHost/HostNameProvider.cs renamed to ‎src/WebJobs.Script/HostNameProvider.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
using Microsoft.AspNetCore.Http;
66
using Microsoft.Extensions.Logging;
77

8-
namespace Microsoft.Azure.WebJobs.Script.WebHost
8+
namespace Microsoft.Azure.WebJobs.Script
99
{
1010
/// <summary>
1111
/// Provides the current HostName for the Function App.

‎src/WebJobs.Script/Properties/Resources.Designer.cs

Lines changed: 9 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/WebJobs.Script/Properties/Resources.resx

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,4 +164,7 @@
164164
IsStopwatchHighResolution: {3}
165165
}}</value>
166166
</data>
167+
<data name="HostIdCollisionFormat" xml:space="preserve">
168+
<value>A collision for Host ID '{0}' was detected in the configured storage account. For more information, see https://aka.ms/functions-hostid-collision.</value>
169+
</data>
167170
</root>

‎src/WebJobs.Script/ScriptHostBuilderExtensions.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -330,6 +330,8 @@ public static void AddCommonServices(IServiceCollection services)
330330
// to be careful with caching, etc. E.g. these services will get
331331
// initially created in placeholder mode and live on through the
332332
// specialized app.
333+
services.AddSingleton<HostNameProvider>();
334+
services.AddSingleton<HostIdValidator>();
333335
services.AddSingleton<IHostIdProvider, ScriptHostIdProvider>();
334336
services.TryAddSingleton<IScriptEventManager, ScriptEventManager>();
335337

0 commit comments

Comments
 (0)