Skip to content

Commit eb9cffb

Browse files
suhsteveimback82
suhsteve
authored and
imback82
committed
Load Assembly referenced from within a lambda closure (#135)
1 parent 815c2a9 commit eb9cffb

File tree

6 files changed

+405
-31
lines changed

6 files changed

+405
-31
lines changed

‎azure-pipelines.yml

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -55,14 +55,14 @@ jobs:
5555
inputs:
5656
command: test
5757
projects: '**/*UnitTest/*.csproj'
58-
arguments: '--configuration $(buildConfiguration) /p:CollectCoverage=true /p:CoverletOutputFormat=cobertura'
58+
arguments: '--configuration $(buildConfiguration)'
5959

6060
- task: DotNetCoreCLI@2
6161
displayName: 'E2E tests for Spark 2.3.0'
6262
inputs:
6363
command: test
6464
projects: '**/Microsoft.Spark.E2ETest/*.csproj'
65-
arguments: '--configuration $(buildConfiguration) /p:CollectCoverage=true /p:CoverletOutputFormat=cobertura'
65+
arguments: '--configuration $(buildConfiguration)'
6666
env:
6767
SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.0-bin-hadoop2.7
6868
HADOOP_HOME: $(Build.BinariesDirectory)\hadoop
@@ -73,7 +73,7 @@ jobs:
7373
inputs:
7474
command: test
7575
projects: '**/Microsoft.Spark.E2ETest/*.csproj'
76-
arguments: '--configuration $(buildConfiguration) /p:CollectCoverage=true /p:CoverletOutputFormat=cobertura'
76+
arguments: '--configuration $(buildConfiguration)'
7777
env:
7878
SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.1-bin-hadoop2.7
7979
HADOOP_HOME: $(Build.BinariesDirectory)\hadoop
@@ -84,7 +84,7 @@ jobs:
8484
inputs:
8585
command: test
8686
projects: '**/Microsoft.Spark.E2ETest/*.csproj'
87-
arguments: '--configuration $(buildConfiguration) /p:CollectCoverage=true /p:CoverletOutputFormat=cobertura'
87+
arguments: '--configuration $(buildConfiguration)'
8888
env:
8989
SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.2-bin-hadoop2.7
9090
HADOOP_HOME: $(Build.BinariesDirectory)\hadoop
@@ -95,7 +95,7 @@ jobs:
9595
inputs:
9696
command: test
9797
projects: '**/Microsoft.Spark.E2ETest/*.csproj'
98-
arguments: '--configuration $(buildConfiguration) /p:CollectCoverage=true /p:CoverletOutputFormat=cobertura'
98+
arguments: '--configuration $(buildConfiguration)'
9999
env:
100100
SPARK_HOME: $(Build.BinariesDirectory)\spark-2.3.3-bin-hadoop2.7
101101
HADOOP_HOME: $(Build.BinariesDirectory)\hadoop
@@ -106,7 +106,7 @@ jobs:
106106
inputs:
107107
command: test
108108
projects: '**/Microsoft.Spark.E2ETest/*.csproj'
109-
arguments: '--configuration $(buildConfiguration) /p:CollectCoverage=true /p:CoverletOutputFormat=cobertura'
109+
arguments: '--configuration $(buildConfiguration)'
110110
env:
111111
SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.0-bin-hadoop2.7
112112
HADOOP_HOME: $(Build.BinariesDirectory)\hadoop
@@ -117,7 +117,7 @@ jobs:
117117
inputs:
118118
command: test
119119
projects: '**/Microsoft.Spark.E2ETest/*.csproj'
120-
arguments: '--configuration $(buildConfiguration) /p:CollectCoverage=true /p:CoverletOutputFormat=cobertura'
120+
arguments: '--configuration $(buildConfiguration)'
121121
env:
122122
SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.1-bin-hadoop2.7
123123
HADOOP_HOME: $(Build.BinariesDirectory)\hadoop
@@ -128,7 +128,7 @@ jobs:
128128
inputs:
129129
command: test
130130
projects: '**/Microsoft.Spark.E2ETest/*.csproj'
131-
arguments: '--configuration $(buildConfiguration) /p:CollectCoverage=true /p:CoverletOutputFormat=cobertura'
131+
arguments: '--configuration $(buildConfiguration)'
132132
env:
133133
SPARK_HOME: $(Build.BinariesDirectory)\spark-2.4.3-bin-hadoop2.7
134134
HADOOP_HOME: $(Build.BinariesDirectory)\hadoop

‎src/csharp/Microsoft.Spark.UnitTest/UdfSerDeTests.cs

Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
using System;
6+
using System.IO;
7+
using System.Reflection;
8+
using System.Runtime.Serialization.Formatters.Binary;
9+
using Microsoft.Spark.Utils;
10+
using Xunit;
11+
12+
namespace Microsoft.Spark.UnitTest
13+
{
14+
public class UdfSerDeTests
15+
{
16+
[Serializable]
17+
private class TestClass
18+
{
19+
private string _str;
20+
21+
public TestClass(string s)
22+
{
23+
_str = s;
24+
}
25+
26+
public string Concat(string s)
27+
{
28+
if (_str == null)
29+
{
30+
return s + s;
31+
}
32+
33+
return _str + s;
34+
}
35+
36+
public override bool Equals(object obj)
37+
{
38+
var that = obj as TestClass;
39+
40+
if (that == null)
41+
{
42+
return false;
43+
}
44+
45+
return _str == that._str;
46+
}
47+
48+
public override int GetHashCode()
49+
{
50+
return base.GetHashCode();
51+
}
52+
}
53+
54+
[Fact]
55+
public void TestUdfSerDe()
56+
{
57+
{
58+
// Without closure.
59+
Func<int, int> expectedUdf = i => 10 * i;
60+
Delegate actualUdf = SerDe(expectedUdf);
61+
62+
VerifyUdfSerDe(expectedUdf, actualUdf, false);
63+
Assert.Equal(100, ((Func<int, int>)actualUdf)(10));
64+
}
65+
66+
{
67+
// With closure where the delegate target is an anonymous class.
68+
// The target will contain fields ["tc1", "tc2"], where "tc1" is
69+
// non null and "tc2" is null.
70+
TestClass tc1 = new TestClass("Test");
71+
TestClass tc2 = null;
72+
Func<string, string> expectedUdf =
73+
(s) =>
74+
{
75+
if (tc2 == null)
76+
{
77+
return tc1.Concat(s);
78+
}
79+
return s;
80+
};
81+
Delegate actualUdf = SerDe(expectedUdf);
82+
83+
VerifyUdfSerDe(expectedUdf, actualUdf, true);
84+
Assert.Equal("TestHelloWorld", ((Func<string, string>)actualUdf)("HelloWorld"));
85+
}
86+
87+
{
88+
// With closure where the delegate target is TestClass
89+
// and target's field "_str" is set to "Test".
90+
TestClass tc = new TestClass("Test");
91+
Func<string, string> expectedUdf = tc.Concat;
92+
Delegate actualUdf = SerDe(expectedUdf);
93+
94+
VerifyUdfSerDe(expectedUdf, actualUdf, true);
95+
Assert.Equal("TestHelloWorld", ((Func<string, string>)actualUdf)("HelloWorld"));
96+
}
97+
98+
{
99+
// With closure where the delegate target is TestClass,
100+
// and target's field "_str" is set to null.
101+
TestClass tc = new TestClass(null);
102+
Func<string, string> expectedUdf = tc.Concat;
103+
Delegate actualUdf = SerDe(expectedUdf);
104+
105+
VerifyUdfSerDe(expectedUdf, actualUdf, true);
106+
Assert.Equal(
107+
"HelloWorldHelloWorld",
108+
((Func<string, string>)actualUdf)("HelloWorld"));
109+
}
110+
}
111+
112+
private void VerifyUdfSerDe(Delegate expectedUdf, Delegate actualUdf, bool hasClosure)
113+
{
114+
VerifyUdfData(
115+
UdfSerDe.Serialize(expectedUdf),
116+
UdfSerDe.Serialize(actualUdf),
117+
hasClosure);
118+
VerifyDelegate(expectedUdf, actualUdf);
119+
}
120+
121+
private void VerifyUdfData(
122+
UdfSerDe.UdfData expectedUdfData,
123+
UdfSerDe.UdfData actualUdfData,
124+
bool hasClosure)
125+
{
126+
Assert.Equal(expectedUdfData, actualUdfData);
127+
128+
if (!hasClosure)
129+
{
130+
Assert.Null(expectedUdfData.TargetData.Fields);
131+
Assert.Null(actualUdfData.TargetData.Fields);
132+
}
133+
}
134+
135+
private void VerifyDelegate(Delegate expectedDelegate, Delegate actualDelegate)
136+
{
137+
Assert.Equal(expectedDelegate.GetType(), actualDelegate.GetType());
138+
Assert.Equal(expectedDelegate.Method, actualDelegate.Method);
139+
Assert.Equal(expectedDelegate.Target.GetType(), actualDelegate.Target.GetType());
140+
141+
FieldInfo[] expectedFields = expectedDelegate.Target.GetType().GetFields();
142+
FieldInfo[] actualFields = actualDelegate.Target.GetType().GetFields();
143+
Assert.Equal(expectedFields, actualFields);
144+
}
145+
146+
private Delegate SerDe(Delegate udf)
147+
{
148+
return Deserialize(Serialize(udf));
149+
}
150+
151+
private byte[] Serialize(Delegate udf)
152+
{
153+
UdfSerDe.UdfData udfData = UdfSerDe.Serialize(udf);
154+
155+
using (var ms = new MemoryStream())
156+
{
157+
var bf = new BinaryFormatter();
158+
bf.Serialize(ms, udfData);
159+
return ms.ToArray();
160+
}
161+
}
162+
163+
private Delegate Deserialize(byte[] serializedUdf)
164+
{
165+
using (var ms = new MemoryStream(serializedUdf, false))
166+
{
167+
var bf = new BinaryFormatter();
168+
UdfSerDe.UdfData udfData = (UdfSerDe.UdfData)bf.Deserialize(ms);
169+
return UdfSerDe.Deserialize(udfData);
170+
}
171+
}
172+
}
173+
}

‎src/csharp/Microsoft.Spark.Worker/Microsoft.Spark.Worker.csproj

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,10 @@
1616
<PackageReference Include="System.Memory" Version="4.5.2" />
1717
</ItemGroup>
1818

19+
<ItemGroup Condition=" '$(TargetFramework)' == 'netcoreapp2.1' ">
20+
<PackageReference Include="System.Runtime.Loader" Version="4.3.0" />
21+
</ItemGroup>
22+
1923
<ItemGroup>
2024
<ProjectReference Include="..\Microsoft.Spark\Microsoft.Spark.csproj" />
2125
</ItemGroup>

‎src/csharp/Microsoft.Spark.Worker/Processor/CommandProcessor.cs

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,23 @@
99
using Microsoft.Spark.Utils;
1010
using static Microsoft.Spark.Utils.UdfUtils;
1111

12+
#if NETCOREAPP
13+
using System.Runtime.Loader;
14+
#endif
15+
1216
namespace Microsoft.Spark.Worker.Processor
1317
{
1418
internal sealed class CommandProcessor
1519
{
1620
private readonly Version _version;
1721

22+
#if NETCOREAPP
23+
static CommandProcessor()
24+
{
25+
UdfSerDe.AssemblyLoader = AssemblyLoadContext.Default.LoadFromAssemblyPath;
26+
}
27+
#endif
28+
1829
internal CommandProcessor(Version version)
1930
{
2031
_version = version;

‎src/csharp/Microsoft.Spark/Utils/CommandSerDe.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -212,7 +212,7 @@ private static void SerializeUdfs(
212212

213213
foreach (UdfSerDe.FieldData field in fields)
214214
{
215-
SerializeUdfs((Delegate)field.Value, curNode, udfWrapperNodes, udfs);
215+
SerializeUdfs((Delegate)field.ValueData.Value, curNode, udfWrapperNodes, udfs);
216216
}
217217
}
218218

0 commit comments

Comments
 (0)