Skip to content

Commit 19bb671

Browse files
mathewcmathewc
authored
HostID Collision detection (#7682)
1 parent f64b710 commit 19bb671

File tree

11 files changed

+527
-31
lines changed

11 files changed

+527
-31
lines changed

‎src/WebJobs.Script.WebHost/WebHostServiceCollectionExtensions.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,6 @@ public static void AddWebJobsScriptHost(this IServiceCollection services, IConfi
124124
services.AddSingleton<IInstanceManager, InstanceManager>();
125125
services.AddSingleton(_ => new HttpClient());
126126
services.AddSingleton<StartupContextProvider>();
127-
services.AddSingleton<HostNameProvider>();
128127
services.AddSingleton<IFileSystem>(_ => FileUtility.Instance);
129128
services.AddTransient<VirtualFileSystem>();
130129
services.AddTransient<VirtualFileSystemMiddleware>();

‎src/WebJobs.Script/Environment/EnvironmentSettingNames.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ public static class EnvironmentSettingNames
5858
public const string TestDataCapEnabled = "WEBSITE_FUNCTIONS_TESTDATA_CAP_ENABLED";
5959
public const string AzureMonitorCategories = "WEBSITE_FUNCTIONS_AZUREMONITOR_CATEGORIES";
6060
public const string FunctionsRequestBodySizeLimit = "FUNCTIONS_REQUEST_BODY_SIZE_LIMIT";
61+
public const string FunctionsHostIdCheckLevel = "FUNCTIONS_HOSTID_CHECK_LEVEL";
6162

6263
//Function in Kubernetes
6364
public const string PodNamespace = "POD_NAMESPACE";

‎src/WebJobs.Script/Host/HostIdValidator.cs

Lines changed: 201 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,201 @@
1+
// Copyright (c) .NET Foundation. All rights reserved.
2+
// Licensed under the MIT License. See License.txt in the project root for license information.
3+
4+
using System;
5+
using System.IO;
6+
using System.Threading.Tasks;
7+
using Azure;
8+
using Azure.Storage.Blobs;
9+
using Microsoft.Azure.WebJobs.Script.Properties;
10+
using Microsoft.Extensions.Logging;
11+
using Newtonsoft.Json;
12+
using IApplicationLifetime = Microsoft.AspNetCore.Hosting.IApplicationLifetime;
13+
14+
namespace Microsoft.Azure.WebJobs.Script
15+
{
16+
/// <summary>
17+
/// Used to perform Host ID validation checks, ensuring that when hosts are sharing
18+
/// a storage account, their computed IDs don't collide.
19+
/// </summary>
20+
/// <remarks>
21+
/// <see cref="ScriptHostIdProvider"/> computes a Host ID and truncates it if needed to
22+
/// ensure it's under length limits. For two different Function Apps, this can result in
23+
/// both apps resolving to the same Host ID. This can cause problems if those apps share
24+
/// a storage account. This class helps detect/prevent such cases.
25+
/// </remarks>
26+
public class HostIdValidator
27+
{
28+
public const string BlobPathFormat = "ids/usage/{0}";
29+
private const LogLevel DefaultLevel = LogLevel.Warning;
30+
31+
private readonly IEnvironment _environment;
32+
private readonly IAzureStorageProvider _storageProvider;
33+
private readonly IApplicationLifetime _applicationLifetime;
34+
private readonly HostNameProvider _hostNameProvider;
35+
private readonly ILogger _logger;
36+
37+
private readonly object _syncLock = new object();
38+
private bool _validationScheduled;
39+
40+
public HostIdValidator(IEnvironment environment, IAzureStorageProvider storageProvider, IApplicationLifetime applicationLifetime,
41+
HostNameProvider hostNameProvider, ILogger<HostIdValidator> logger)
42+
{
43+
_environment = environment;
44+
_storageProvider = storageProvider;
45+
_applicationLifetime = applicationLifetime;
46+
_hostNameProvider = hostNameProvider;
47+
_logger = logger;
48+
}
49+
50+
internal bool ValidationScheduled => _validationScheduled;
51+
52+
public virtual void ScheduleValidation(string hostId)
53+
{
54+
lock (_syncLock)
55+
{
56+
if (!_validationScheduled)
57+
{
58+
// Schedule the validation to run asynchronously after a delay. This delay ensures
59+
// we're not impacting coldstart.
60+
Utility.ExecuteAfterColdStartDelay(_environment, () => Task.Run(() => ValidateHostIdUsageAsync(hostId)));
61+
_validationScheduled = true;
62+
}
63+
}
64+
}
65+
66+
internal async Task ValidateHostIdUsageAsync(string hostId)
67+
{
68+
try
69+
{
70+
if (!_storageProvider.ConnectionExists(ConnectionStringNames.Storage))
71+
{
72+
return;
73+
}
74+
75+
HostIdInfo hostIdInfo = await ReadHostIdInfoAsync(hostId);
76+
77+
if (hostIdInfo != null)
78+
{
79+
// an existing record exists for this host ID
80+
CheckForCollision(hostId, hostIdInfo);
81+
}
82+
else
83+
{
84+
// no existing record, so write one, claiming this host ID for this host name
85+
// in this storage account
86+
hostIdInfo = new HostIdInfo
87+
{
88+
Hostname = _hostNameProvider.Value
89+
};
90+
await WriteHostIdAsync(hostId, hostIdInfo);
91+
}
92+
}
93+
catch (Exception ex)
94+
{
95+
// best effort - log error and continue
96+
_logger.LogError(ex, "Error validating host ID usage.");
97+
}
98+
}
99+
100+
private void CheckForCollision(string hostId, HostIdInfo hostIdInfo)
101+
{
102+
// verify the host name is the same as our host name
103+
if (!string.Equals(_hostNameProvider.Value, hostIdInfo.Hostname, StringComparison.OrdinalIgnoreCase))
104+
{
105+
HandleCollision(hostId);
106+
}
107+
}
108+
109+
private void HandleCollision(string hostId)
110+
{
111+
// see if the user has specified a level, otherwise default
112+
string value = _environment.GetEnvironmentVariable(EnvironmentSettingNames.FunctionsHostIdCheckLevel);
113+
if (!Enum.TryParse<LogLevel>(value, out LogLevel level))
114+
{
115+
level = DefaultLevel;
116+
}
117+
118+
string message = string.Format(Resources.HostIdCollisionFormat, hostId);
119+
if (level == LogLevel.Error)
120+
{
121+
_logger.LogError(message);
122+
_applicationLifetime.StopApplication();
123+
}
124+
else
125+
{
126+
// we only allow Warning/Error levels to be specified, so anything other than
127+
// Error is treated as warning
128+
_logger.LogWarning(message);
129+
}
130+
}
131+
132+
internal async Task WriteHostIdAsync(string hostId, HostIdInfo hostIdInfo)
133+
{
134+
try
135+
{
136+
var containerClient = _storageProvider.GetBlobContainerClient();
137+
string blobPath = string.Format(BlobPathFormat, hostId);
138+
BlobClient blobClient = containerClient.GetBlobClient(blobPath);
139+
BinaryData data = BinaryData.FromObjectAsJson(hostIdInfo);
140+
await blobClient.UploadAsync(data);
141+
142+
_logger.LogDebug($"Host ID record written (ID:{hostId}, HostName:{hostIdInfo.Hostname})");
143+
}
144+
catch (RequestFailedException rfex) when (rfex.Status == 409)
145+
{
146+
// Another instance wrote the blob between the time when we initially
147+
// checked and when we attempted to write. Read the blob and validate it.
148+
hostIdInfo = await ReadHostIdInfoAsync(hostId);
149+
if (hostIdInfo != null)
150+
{
151+
CheckForCollision(hostId, hostIdInfo);
152+
}
153+
}
154+
catch (Exception ex)
155+
{
156+
// best effort
157+
_logger.LogError(ex, "Error writing host ID info");
158+
}
159+
}
160+
161+
internal async Task<HostIdInfo> ReadHostIdInfoAsync(string hostId)
162+
{
163+
HostIdInfo hostIdInfo = null;
164+
165+
try
166+
{
167+
// check storage to see if a record already exists for this host ID
168+
var containerClient = _storageProvider.GetBlobContainerClient();
169+
string blobPath = string.Format(BlobPathFormat, hostId);
170+
BlobClient blobClient = containerClient.GetBlobClient(blobPath);
171+
var downloadResponse = await blobClient.DownloadAsync();
172+
string content;
173+
using (StreamReader reader = new StreamReader(downloadResponse.Value.Content))
174+
{
175+
content = reader.ReadToEnd();
176+
}
177+
178+
if (!string.IsNullOrEmpty(content))
179+
{
180+
hostIdInfo = JsonConvert.DeserializeObject<HostIdInfo>(content);
181+
}
182+
}
183+
catch (RequestFailedException exception) when (exception.Status == 404)
184+
{
185+
// no record stored for this host ID
186+
}
187+
catch (Exception ex)
188+
{
189+
// best effort
190+
_logger.LogError(ex, "Error reading host ID info");
191+
}
192+
193+
return hostIdInfo;
194+
}
195+
196+
internal class HostIdInfo
197+
{
198+
public string Hostname { get; set; }
199+
}
200+
}
201+
}

‎src/WebJobs.Script/Host/ScriptHost.cs

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,6 @@ public class ScriptHost : JobHost, IScriptJobHost
4646
internal const string GeneratedTypeName = "Functions";
4747
private readonly IApplicationLifetime _applicationLifetime;
4848
private readonly IScriptHostManager _scriptHostManager;
49-
private readonly string _storageConnectionString;
5049
private readonly IDistributedLockManager _distributedLockManager;
5150
private readonly IFunctionMetadataManager _functionMetadataManager;
5251
private readonly IFileLoggingStatusManager _fileLoggingStatusManager;
@@ -118,7 +117,6 @@ public ScriptHost(IOptions<JobHostOptions> options,
118117
_instanceId = Guid.NewGuid().ToString();
119118
_hostOptions = options;
120119
_configuration = configuration;
121-
_storageConnectionString = configuration.GetWebJobsConnectionString(ConnectionStringNames.Storage);
122120
_distributedLockManager = distributedLockManager;
123121
_functionMetadataManager = functionMetadataManager;
124122
_fileLoggingStatusManager = fileLoggingStatusManager;

‎src/WebJobs.Script/Host/ScriptHostIdProvider.cs

Lines changed: 33 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,21 +18,36 @@ public class ScriptHostIdProvider : IHostIdProvider
1818
private readonly IConfiguration _config;
1919
private readonly IEnvironment _environment;
2020
private readonly IOptionsMonitor<ScriptApplicationHostOptions> _options;
21+
private readonly HostIdValidator _hostIdValidator;
2122

22-
public ScriptHostIdProvider(IConfiguration config, IEnvironment environment, IOptionsMonitor<ScriptApplicationHostOptions> options)
23+
public ScriptHostIdProvider(IConfiguration config, IEnvironment environment, IOptionsMonitor<ScriptApplicationHostOptions> options, HostIdValidator hostIdValidator)
2324
{
2425
_config = config;
2526
_environment = environment;
2627
_options = options;
28+
_hostIdValidator = hostIdValidator;
2729
}
2830

2931
public Task<string> GetHostIdAsync(CancellationToken cancellationToken)
3032
{
31-
return Task.FromResult(_config[ConfigurationSectionNames.HostIdPath] ?? GetDefaultHostId(_environment, _options.CurrentValue));
33+
string hostId = _config[ConfigurationSectionNames.HostIdPath];
34+
if (hostId == null)
35+
{
36+
HostIdResult result = GetDefaultHostId(_environment, _options.CurrentValue);
37+
hostId = result.HostId;
38+
if (result.IsTruncated && !result.IsLocal)
39+
{
40+
_hostIdValidator.ScheduleValidation(hostId);
41+
}
42+
}
43+
44+
return Task.FromResult(hostId);
3245
}
3346

34-
internal static string GetDefaultHostId(IEnvironment environment, ScriptApplicationHostOptions scriptOptions)
47+
internal static HostIdResult GetDefaultHostId(IEnvironment environment, ScriptApplicationHostOptions scriptOptions)
3548
{
49+
HostIdResult result = new HostIdResult();
50+
3651
// We're setting the default here on the newly created configuration
3752
// If the user has explicitly set the HostID via host.json, it will overwrite
3853
// what we set here
@@ -64,19 +79,32 @@ internal static string GetDefaultHostId(IEnvironment environment, ScriptApplicat
6479
.Where(char.IsLetterOrDigit)
6580
.Aggregate(new StringBuilder(), (b, c) => b.Append(c)).ToString();
6681
hostId = $"{sanitizedMachineName}-{Math.Abs(Utility.GetStableHash(scriptOptions.ScriptPath))}";
82+
result.IsLocal = true;
6783
}
6884

6985
if (!string.IsNullOrEmpty(hostId))
7086
{
7187
if (hostId.Length > ScriptConstants.MaximumHostIdLength)
7288
{
73-
// Truncate to the max host name length if needed
89+
// Truncate to the max host name length
7490
hostId = hostId.Substring(0, ScriptConstants.MaximumHostIdLength);
91+
result.IsTruncated = true;
7592
}
7693
}
7794

7895
// Lowercase and trim any trailing '-' as they can cause problems with queue names
79-
return hostId?.ToLowerInvariant().TrimEnd('-');
96+
result.HostId = hostId?.ToLowerInvariant().TrimEnd('-');
97+
98+
return result;
99+
}
100+
101+
public class HostIdResult
102+
{
103+
public string HostId { get; set; }
104+
105+
public bool IsTruncated { get; set; }
106+
107+
public bool IsLocal { get; set; }
80108
}
81109
}
82110
}

‎src/WebJobs.Script.WebHost/HostNameProvider.cs renamed to ‎src/WebJobs.Script/HostNameProvider.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
using Microsoft.AspNetCore.Http;
66
using Microsoft.Extensions.Logging;
77

8-
namespace Microsoft.Azure.WebJobs.Script.WebHost
8+
namespace Microsoft.Azure.WebJobs.Script
99
{
1010
/// <summary>
1111
/// Provides the current HostName for the Function App.

‎src/WebJobs.Script/Properties/Resources.Designer.cs

Lines changed: 9 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/WebJobs.Script/Properties/Resources.resx

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,4 +164,7 @@
164164
IsStopwatchHighResolution: {3}
165165
}}</value>
166166
</data>
167+
<data name="HostIdCollisionFormat" xml:space="preserve">
168+
<value>A collision for Host ID '{0}' was detected in the configured storage account. For more information, see https://aka.ms/functions-hostid-collision.</value>
169+
</data>
167170
</root>

‎src/WebJobs.Script/ScriptHostBuilderExtensions.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -330,6 +330,8 @@ public static void AddCommonServices(IServiceCollection services)
330330
// to be careful with caching, etc. E.g. these services will get
331331
// initially created in placeholder mode and live on through the
332332
// specialized app.
333+
services.AddSingleton<HostNameProvider>();
334+
services.AddSingleton<HostIdValidator>();
333335
services.AddSingleton<IHostIdProvider, ScriptHostIdProvider>();
334336
services.TryAddSingleton<IScriptEventManager, ScriptEventManager>();
335337

0 commit comments

Comments
 (0)