Skip to content

Commit 2c72e8c

Browse files
asmirnov82asmirnov82
committed
Merge branch 'main' into 6182_load_csv_duplicate_columns
2 parents ecb9114 + 39235a7 commit 2c72e8c

28 files changed

+1880
-1745
lines changed

‎build/libomp.rb

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ class Libomp < Formula
1515
end
1616

1717
depends_on "cmake" => :build
18-
depends_on macos: :yosemite
1918

2019
def install
2120
system "cmake", ".", *std_cmake_args

‎src/Microsoft.Data.Analysis/DataFrame.IO.cs

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -459,14 +459,13 @@ private static DataFrame ReadCsvLinesIntoDataFrame(WrappedStreamReaderOrStringRe
459459

460460
if (addIndexColumn)
461461
{
462-
PrimitiveDataFrameColumn<int> indexColumn = new PrimitiveDataFrameColumn<int>("IndexColumn", columns[0].Length);
463-
for (int i = 0; i < columns[0].Length; i++)
462+
Int64DataFrameColumn indexColumn = new Int64DataFrameColumn("IndexColumn", columns[0].Length);
463+
for (long i = 0; i < columns[0].Length; i++)
464464
{
465465
indexColumn[i] = i;
466466
}
467-
columns.Insert(0, indexColumn);
467+
ret.Columns.Insert(0, indexColumn);
468468
}
469-
470469
}
471470

472471
return ret;

‎src/Microsoft.ML.AutoML/CodeGen/estimator-schema.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,8 @@
7373
"ForecastBySsa",
7474
"TextClassifcation",
7575
"SentenceSimilarity",
76-
"ObjectDetection"
76+
"ObjectDetection",
77+
"QuestionAnswering"
7778
]
7879
},
7980
"nugetDependencies": {

‎src/Microsoft.ML.AutoML/CodeGen/question_answering_search_space.json

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
{
2+
"$schema": "./search-space-schema.json#",
3+
"name": "question_answering_option",
4+
"search_space": [
5+
{
6+
"name": "ContextColumnName",
7+
"type": "string",
8+
"default": "Context"
9+
},
10+
{
11+
"name": "QuestionColumnName",
12+
"type": "string",
13+
"default": "Question"
14+
},
15+
{
16+
"name": "TrainingAnswerColumnName",
17+
"type": "string",
18+
"default": "TrainingAnswer"
19+
},
20+
{
21+
"name": "AnswerIndexStartColumnName",
22+
"type": "string",
23+
"default": "AnswerStart"
24+
},
25+
{
26+
"name": "ScoreColumnName",
27+
"type": "string",
28+
"default": "Score"
29+
},
30+
{
31+
"name": "predictedAnswerColumnName",
32+
"type": "string",
33+
"default": "Answer"
34+
},
35+
{
36+
"name": "BatchSize",
37+
"type": "integer",
38+
"default": 4
39+
},
40+
{
41+
"name": "MaxEpochs",
42+
"type": "integer",
43+
"default": 10
44+
},
45+
{
46+
"name": "TopKAnswers",
47+
"type": "integer",
48+
"default": 3
49+
},
50+
{
51+
"name": "Architecture",
52+
"type": "bertArchitecture",
53+
"default": "BertArchitecture.Roberta"
54+
}
55+
]
56+
}

‎src/Microsoft.ML.AutoML/CodeGen/search-space-schema.json

Lines changed: 32 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,26 @@
4646
"BertArchitecture.Roberta"
4747
]
4848
},
49+
"dataKind": {
50+
"type": "string",
51+
"enum": [
52+
"DataKind.Int16",
53+
"DataKind.SByte",
54+
"DataKind.Byte",
55+
"DataKind.UInt16",
56+
"DataKind.Int32",
57+
"DataKind.UInt32",
58+
"DataKind.Int64",
59+
"DataKind.UInt64",
60+
"DataKind.Single",
61+
"DataKind.Double",
62+
"DataKind.String",
63+
"DataKind.Boolean",
64+
"DataKind.TimeSpan",
65+
"DataKind.DateTime",
66+
"DataKind.DateTimeOffset"
67+
]
68+
},
4969
"bertArchitectureArray": {
5070
"type": "array",
5171
"items": {
@@ -146,7 +166,8 @@
146166
"dnn_featurizer_image_option",
147167
"text_classification_option",
148168
"sentence_similarity_option",
149-
"object_detection_option"
169+
"object_detection_option",
170+
"question_answering_option"
150171
]
151172
},
152173
"option_name": {
@@ -210,7 +231,14 @@
210231
"Steps",
211232
"MaxEpoch",
212233
"InitLearningRate",
213-
"WeightDecay"
234+
"WeightDecay",
235+
"ContextColumnName",
236+
"QuestionColumnName",
237+
"TrainingAnswerColumnName",
238+
"AnswerIndexStartColumnName",
239+
"predictedAnswerColumnName",
240+
"TopKAnswers",
241+
"TargetType"
214242
]
215243
},
216244
"option_type": {
@@ -228,7 +256,8 @@
228256
"anchor",
229257
"dnnModelFactory",
230258
"bertArchitecture",
231-
"imageClassificationArchType"
259+
"imageClassificationArchType",
260+
"dataKind"
232261
]
233262
}
234263
},

‎src/Microsoft.ML.AutoML/CodeGen/trainer-estimators.json

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -532,6 +532,13 @@
532532
"usingStatements": [ "Microsoft.ML", "Microsoft.ML.Trainers", "Microsoft.ML.TorchSharp" ],
533533
"searchOption": "object_detection_option"
534534
},
535+
{
536+
"functionName": "QuestionAnswering",
537+
"estimatorTypes": [ "MultiClassification" ],
538+
"nugetDependencies": [ "Microsoft.ML", "Microsoft.ML.TorchSharp" ],
539+
"usingStatements": [ "Microsoft.ML", "Microsoft.ML.Trainers", "Microsoft.ML.TorchSharp" ],
540+
"searchOption": "question_answering_option"
541+
},
535542
{
536543
"functionName": "ForecastBySsa",
537544
"estimatorTypes": [ "Forecasting" ],

‎src/Microsoft.ML.AutoML/CodeGen/type_converter_search_space.json

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,11 @@
99
{
1010
"name": "InputColumnNames",
1111
"type": "strings"
12+
},
13+
{
14+
"name": "TargetType",
15+
"type": "dataKind",
16+
"default": "DataKind.Single"
1217
}
1318
]
1419
}

‎src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@
6969
<AdditionalFiles Include="CodeGen\code_gen_flag.json" />
7070
<AdditionalFiles Include="CodeGen\*-estimators.json" />
7171
</ItemGroup>
72-
72+
7373
<ItemGroup>
7474
<EmbeddedResource Include="Tuner\Portfolios.json">
7575
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>

‎src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/QuestionAnswering.cs

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
using Microsoft.ML.TorchSharp;
5+
using Microsoft.ML.TorchSharp.NasBert;
6+
using Microsoft.ML.TorchSharp.Roberta;
7+
8+
namespace Microsoft.ML.AutoML.CodeGen
9+
{
10+
internal partial class QuestionAnsweringMulti
11+
{
12+
public override IEstimator<ITransformer> BuildFromOption(MLContext context, QuestionAnsweringOption param)
13+
{
14+
return context.MulticlassClassification.Trainers.QuestionAnswer(
15+
contextColumnName: param.ContextColumnName,
16+
questionColumnName: param.QuestionColumnName,
17+
trainingAnswerColumnName: param.TrainingAnswerColumnName,
18+
answerIndexColumnName: param.AnswerIndexStartColumnName,
19+
predictedAnswerColumnName: param.PredictedAnswerColumnName,
20+
scoreColumnName: param.ScoreColumnName,
21+
batchSize: param.BatchSize,
22+
maxEpochs: param.MaxEpochs,
23+
topK: param.TopKAnswers,
24+
architecture: BertArchitecture.Roberta);
25+
}
26+
27+
}
28+
}

‎src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/TypeConvert.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ internal partial class ConvertType
99
public override IEstimator<ITransformer> BuildFromOption(MLContext context, ConvertTypeOption param)
1010
{
1111
var inputOutputPairs = AutoMlUtils.CreateInputOutputColumnPairsFromStrings(param.InputColumnNames, param.OutputColumnNames);
12-
return context.Transforms.Conversion.ConvertType(inputOutputPairs);
12+
return context.Transforms.Conversion.ConvertType(inputOutputPairs, param.TargetType);
1313
}
1414
}
1515
}

‎src/Microsoft.ML.Core/ComponentModel/ComponentCatalog.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ internal static AccessModifier Accessmodifier(this MethodInfo methodInfo)
3030
return AccessModifier.Internal;
3131
if (methodInfo.IsPublic)
3232
return AccessModifier.Public;
33-
throw new ArgumentException("Did not find access modifier", "methodInfo");
33+
throw new ArgumentException("Did not find access modifier", nameof(methodInfo));
3434
}
3535

3636
internal static AccessModifier Accessmodifier(this ConstructorInfo constructorInfo)
@@ -47,7 +47,7 @@ internal static AccessModifier Accessmodifier(this ConstructorInfo constructorIn
4747
return AccessModifier.Internal;
4848
if (constructorInfo.IsPublic)
4949
return AccessModifier.Public;
50-
throw new ArgumentException("Did not find access modifier", "constructorInfo");
50+
throw new ArgumentException("Did not find access modifier", nameof(constructorInfo));
5151
}
5252

5353
internal enum AccessModifier

‎src/Microsoft.ML.Core/EntryPoints/EntryPointModuleAttribute.cs

Lines changed: 17 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -3,23 +3,22 @@
33
// See the LICENSE file in the project root for more information.
44

55
using System;
6-
namespace Microsoft.ML.EntryPoints
7-
{
8-
/// <summary>
9-
/// This is a signature for classes that are 'holders' of entry points and components.
10-
/// </summary>
11-
[BestFriend]
12-
internal delegate void SignatureEntryPointModule();
6+
namespace Microsoft.ML.EntryPoints;
7+
8+
/// <summary>
9+
/// This is a signature for classes that are 'holders' of entry points and components.
10+
/// </summary>
11+
[BestFriend]
12+
internal delegate void SignatureEntryPointModule();
1313

14-
/// <summary>
15-
/// A simplified assembly attribute for marking EntryPoint modules.
16-
/// </summary>
17-
[AttributeUsage(AttributeTargets.Assembly, AllowMultiple = true)]
18-
[BestFriend]
19-
internal sealed class EntryPointModuleAttribute : LoadableClassAttributeBase
20-
{
21-
public EntryPointModuleAttribute(Type loaderType)
22-
: base(null, typeof(void), loaderType, null, new[] { typeof(SignatureEntryPointModule) }, loaderType.FullName)
23-
{ }
24-
}
14+
/// <summary>
15+
/// A simplified assembly attribute for marking EntryPoint modules.
16+
/// </summary>
17+
[AttributeUsage(AttributeTargets.Assembly, AllowMultiple = true)]
18+
[BestFriend]
19+
internal sealed class EntryPointModuleAttribute : LoadableClassAttributeBase
20+
{
21+
public EntryPointModuleAttribute(Type loaderType)
22+
: base(null, typeof(void), loaderType, null, new[] { typeof(SignatureEntryPointModule) }, loaderType.FullName)
23+
{ }
2524
}

0 commit comments

Comments
 (0)