Skip to content

Commit 30a9bf4

Browse files
HLWeilHLWeil
authored
Merge pull request #353 from nfdi4plants/jsonLD-fix
Increase robustness for table<->process<->Json-LD parsing
2 parents 022c7a4 + 8c82636 commit 30a9bf4

10 files changed

Lines changed: 276 additions & 40 deletions

File tree

‎src/Core/Conversion.fs‎

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -68,32 +68,36 @@ module JsonTypes =
6868
/// Convert a CompositeCell to a ISA Value and Unit tuple.
6969
let valueOfCell (value : CompositeCell) =
7070
match value with
71-
| CompositeCell.FreeText (text) -> Value.fromString text, None
72-
| CompositeCell.Term (term) -> Value.Ontology term, None
73-
| CompositeCell.Unitized (text,unit) -> Value.fromString text, Some unit
71+
| CompositeCell.FreeText ("") -> None, None
72+
| CompositeCell.FreeText (text) -> Some (Value.fromString text), None
73+
| CompositeCell.Term (term) when term.isEmpty() -> None, None
74+
| CompositeCell.Term (term) -> Some (Value.Ontology term), None
75+
| CompositeCell.Unitized (text,unit) ->
76+
(if text = "" then None else Value.fromString text |> Some),
77+
if unit.isEmpty() then None else unit |> Some
7478

7579
/// Convert a CompositeHeader and Cell tuple to a ISA Component
7680
let composeComponent (header : CompositeHeader) (value : CompositeCell) : Component =
7781
let v,u = valueOfCell value
78-
Component.create (v,?unit = u,componentType = header.ToTerm())
82+
Component.create (?value = v,?unit = u,componentType = header.ToTerm())
7983

8084
/// Convert a CompositeHeader and Cell tuple to a ISA ProcessParameterValue
8185
let composeParameterValue (header : CompositeHeader) (value : CompositeCell) : ProcessParameterValue =
8286
let v,u = valueOfCell value
8387
let p = ProtocolParameter.create(ParameterName = header.ToTerm())
84-
ProcessParameterValue.create(p,v,?Unit = u)
88+
ProcessParameterValue.create(p,?Value = v,?Unit = u)
8589

8690
/// Convert a CompositeHeader and Cell tuple to a ISA FactorValue
8791
let composeFactorValue (header : CompositeHeader) (value : CompositeCell) : FactorValue =
8892
let v,u = valueOfCell value
8993
let f = Factor.create(Name = header.ToString(),FactorType = header.ToTerm())
90-
FactorValue.create(Category = f,Value = v,?Unit = u)
94+
FactorValue.create(Category = f,?Value = v,?Unit = u)
9195

9296
/// Convert a CompositeHeader and Cell tuple to a ISA MaterialAttributeValue
9397
let composeCharacteristicValue (header : CompositeHeader) (value : CompositeCell) : MaterialAttributeValue =
9498
let v,u = valueOfCell value
9599
let c = MaterialAttribute.create(CharacteristicType = header.ToTerm())
96-
MaterialAttributeValue.create(Category = c,Value = v,?Unit = u)
100+
MaterialAttributeValue.create(Category = c,?Value = v,?Unit = u)
97101

98102
/// Convert a CompositeHeader and Cell tuple to a ISA ProcessInput
99103
let composeProcessInput (header : CompositeHeader) (value : CompositeCell) : ProcessInput =
@@ -124,12 +128,13 @@ module JsonTypes =
124128
let value = value |> Option.defaultValue (Value.Name "")
125129
match value,unit with
126130
| Value.Ontology oa, None -> CompositeCell.Term oa
127-
| Value.Name text, None -> CompositeCell.FreeText text
131+
| Value.Name "", None -> CompositeCell.Term (OntologyAnnotation())
132+
| Value.Name text, None -> CompositeCell.Term (OntologyAnnotation(text))
128133
| Value.Name name, Some u -> CompositeCell.Unitized (name,u)
129134
| Value.Float f, Some u -> CompositeCell.Unitized (f.ToString(),u)
130-
| Value.Float f, None -> CompositeCell.FreeText (f.ToString())
135+
| Value.Float f, None -> CompositeCell.Unitized (f.ToString(),OntologyAnnotation())
131136
| Value.Int i, Some u -> CompositeCell.Unitized (i.ToString(),u)
132-
| Value.Int i, None -> CompositeCell.FreeText (i.ToString())
137+
| Value.Int i, None -> CompositeCell.Unitized (i.ToString(),OntologyAnnotation())
133138
| _ -> failwithf "Could not parse value %O with unit %O" value unit
134139

135140
/// Convert an ISA Component to a CompositeHeader and Cell tuple

‎src/Core/OntologyAnnotation.fs‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -54,14 +54,12 @@ type OntologyAnnotation(?name,?tsr,?tan, ?comments) =
5454
static member createUriAnnotation (termSourceRef : string) (localTAN : string) =
5555
$"{Url.OntobeeOboPurl}{termSourceRef}_{localTAN}"
5656

57-
/// Will always be created without `OntologyAnnotion.Name`
58-
static member fromTermAnnotation (tan : string) =
59-
tan
60-
|> Regex.tryParseTermAnnotation
61-
|> Option.get
62-
|> fun r ->
57+
static member fromTermAnnotation (tan : string, ?name) =
58+
match Regex.tryParseTermAnnotation tan with
59+
| Some r ->
6360
let accession = r.IDSpace + ":" + r.LocalID
64-
OntologyAnnotation.create ("", r.IDSpace, accession)
61+
OntologyAnnotation.create (?name = name, tsr = r.IDSpace, tan = accession)
62+
| None -> OntologyAnnotation.create (?name = name, tan = tan)
6563

6664
/// Parses any value in `TermAccessionString` to term accession format "termsourceref:localtan". Exmp.: "MS:000001".
6765
///
@@ -130,6 +128,9 @@ type OntologyAnnotation(?name,?tsr,?tan, ?comments) =
130128
sb.Append("}") |> ignore
131129
sb.ToString()
132130

131+
member this.isEmpty() =
132+
this.Name.IsNone && this.TermSourceREF.IsNone && this.TermAccessionNumber.IsNone && this.Comments.Count = 0
133+
133134
override this.GetHashCode() =
134135
[|
135136
HashCodes.boxHashOption this.Name

‎src/Core/Process/ColumnIndex.fs‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,12 @@ let createOrderComment (index : int) =
1313
Comment.create(orderName,(string index))
1414

1515
let tryGetIndex (comments : ResizeArray<Comment>) =
16-
comments
17-
|> CommentArray.tryItem orderName
18-
|> Option.bind tryInt
16+
match comments |> CommentArray.tryItem orderName with
17+
| Some ci ->
18+
let i = comments |> Seq.findIndex (fun c -> c.Name = Some orderName)
19+
comments.RemoveAt(i)
20+
tryInt ci
21+
| _ -> None
1922

2023
let setOntologyAnnotationIndexInplace i (oa : OntologyAnnotation) =
2124
oa.Comments.Add(createOrderComment i)

‎src/Core/Process/Value.fs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ type Value =
2626
#endif
2727
match parser value with
2828
| (true, f) -> Value.Float f
29-
| _ -> Value.Ontology <| OntologyAnnotation.create value
29+
| _ -> Value.Name value
3030

3131
static member fromOptions (value : string Option) (termSource: string Option) (termAccesssion: string Option) =
3232
match value, termSource, termAccesssion with

‎src/Json/Process/PropertyValue.fs‎

Lines changed: 46 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,8 @@ module PropertyValue =
1313

1414
let genID (p : IPropertyValue<'T>) =
1515
match p.GetCategory() with
16-
| Some t -> $"{p.GetAdditionalType}/{t}{p.GetValue()}{p.GetUnit()}"
17-
| None -> $"#Empty{p.GetAdditionalType}"
16+
| Some t -> $"{p.GetAdditionalType()}/{t}{p.GetValue()}{p.GetUnit()}"
17+
| None -> $"#Empty{p.GetAdditionalType()}"
1818

1919
let encoder<'T> (pv : IPropertyValue<'T>) =
2020
let categoryName, categoryURL =
@@ -54,20 +54,48 @@ module PropertyValue =
5454

5555
let decoder<'T> (create : createPVFunction<'T>) : Decoder<'T>=
5656
Decode.object (fun get ->
57-
let category =
58-
let name = get.Optional.Field "category" Decode.string
59-
let code = get.Optional.Field "categoryCode" Decode.string
60-
if name.IsNone && code.IsNone then None
61-
else Some(OntologyAnnotation.create(?name = name, ?tan = code))
62-
let value =
63-
let value = get.Optional.Field "value" Decode.string
64-
let code = get.Optional.Field "valueCode" Decode.string
65-
if value.IsNone && code.IsNone then None
66-
else Value.fromOptions value None code
67-
let unit =
68-
let name = get.Optional.Field "unit" Decode.string
69-
let code = get.Optional.Field "unitCode" Decode.string
70-
if name.IsNone && code.IsNone then None
71-
else Some(OntologyAnnotation.create(?name = name, ?tan = code))
72-
create category value unit
57+
58+
let category =
59+
let name = get.Optional.Field "category" Decode.string
60+
let code = get.Optional.Field "categoryCode" Decode.string
61+
match name, code with
62+
| None, None | None, Some "" -> None
63+
| _, None | _, Some "" ->
64+
try Some(OntologyAnnotation.create(?name = name))
65+
with
66+
| err ->
67+
failwith $"Error while decoding category (name:{name}): {err}"
68+
| _, Some code ->
69+
try Some(OntologyAnnotation.fromTermAnnotation(code, ?name = name))
70+
with
71+
| err -> failwith $"Error while decoding category (name:{name}, code:{code}): {err}"
72+
let unit =
73+
let name = get.Optional.Field "unit" Decode.string
74+
let code = get.Optional.Field "unitCode" Decode.string
75+
76+
match name, code with
77+
| None, None | None, Some "" -> None
78+
| _, None | _, Some "" ->
79+
try Some(OntologyAnnotation.create(?name = name))
80+
with
81+
| err ->
82+
failwith $"Error while decoding unit (name:{name}): {err}"
83+
| _, Some code ->
84+
try Some(OntologyAnnotation.fromTermAnnotation(code, ?name = name))
85+
with
86+
| err ->
87+
failwith $"Error while decoding unit (name:{name}, code:{code}): {err}"
88+
89+
let value =
90+
let value = get.Optional.Field "value" AnnotationValue.decoder
91+
let code = get.Optional.Field "valueCode" Decode.string
92+
if value.IsNone && code.IsNone then None
93+
else
94+
try Value.fromOptions value None code
95+
with
96+
| err ->
97+
failwith $"Error while decoding value {value},{code}: {err}"
98+
99+
create category value unit
100+
73101
)

‎src/Json/Study.fs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,7 @@ module Study =
170170
/// <param name="assays"></param>
171171
let encoder (assays: ArcAssay list option) (s : ArcStudy) =
172172
let study = s.Copy(true)
173-
let fileName = Identifier.Assay.fileNameFromIdentifier study.Identifier
173+
let fileName = Identifier.Study.fileNameFromIdentifier study.Identifier
174174
let assaysRaw = Helper.getAssayInformation assays study
175175
let assays =
176176
let n = ResizeArray()

‎tests/Core/ArcJsonConversion.Tests.fs‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -302,6 +302,114 @@ let private tests_ArcTableProcess =
302302
let table = ArcTable.fromProcesses tableName1 processes
303303
Expect.arcTableEqual table t "Table should be equal"
304304
)
305+
ptestCase "ParamValueUnitizedNoValueNoInputOutput" (fun () ->
306+
let t = ArcTable.init tableName1
307+
let header = CompositeHeader.Parameter oa_temperature
308+
let unit = oa_degreeCel
309+
let cell = CompositeCell.createUnitized ("",unit)
310+
t.AddColumn(header,[|cell|])
311+
312+
let processes = t.GetProcesses()
313+
Expect.equal 1 processes.Length "Should have 1 process"
314+
let p = processes.[0]
315+
Expect.equal 1 p.ParameterValues.Value.Length "Should have 1 parameter value"
316+
let pv = p.ParameterValues.Value.[0]
317+
Expect.isNone pv.Value "Should have no value"
318+
let resultCategory = Expect.wantSome pv.Category "Should have category"
319+
Expect.equal resultCategory (ProtocolParameter.create(ParameterName = oa_temperature)) "Category should match"
320+
let resultUnit = Expect.wantSome pv.Unit "Should have unit"
321+
Expect.equal resultUnit unit "Unit should match"
322+
323+
let t' = ArcTable.fromProcesses tableName1 processes
324+
Expect.arcTableEqual t' t "Table should be equal"
325+
)
326+
testCase "ParamValueUnitizedNoValue" (fun () ->
327+
let t = ArcTable.init tableName1
328+
let header = CompositeHeader.Parameter oa_temperature
329+
let unit = oa_degreeCel
330+
let cell = CompositeCell.createUnitized ("",unit)
331+
t.AddColumn(CompositeHeader.Input(IOType.Source),[|CompositeCell.createFreeText "Source"|])
332+
t.AddColumn(header,[|cell|])
333+
t.AddColumn(CompositeHeader.Output(IOType.Sample),[|CompositeCell.createFreeText "Sample"|])
334+
335+
let processes = t.GetProcesses()
336+
Expect.equal 1 processes.Length "Should have 1 process"
337+
let p = processes.[0]
338+
Expect.equal 1 p.ParameterValues.Value.Length "Should have 1 parameter value"
339+
let pv = p.ParameterValues.Value.[0]
340+
Expect.isNone pv.Value "Should have no value"
341+
let resultCategory = Expect.wantSome pv.Category "Should have category"
342+
Expect.equal resultCategory (ProtocolParameter.create(ParameterName = oa_temperature)) "Category should match"
343+
let resultUnit = Expect.wantSome pv.Unit "Should have unit"
344+
Expect.equal resultUnit unit "Unit should match"
345+
346+
let t' = ArcTable.fromProcesses tableName1 processes
347+
Expect.arcTableEqual t' t "Table should be equal"
348+
)
349+
ptestCase "ParamValueUnitizedEmpty" (fun () ->
350+
let t = ArcTable.init tableName1
351+
let header = CompositeHeader.Parameter oa_temperature
352+
let cell = CompositeCell.createUnitized ("")
353+
t.AddColumn(CompositeHeader.Input(IOType.Source),[|CompositeCell.createFreeText "Source"|])
354+
t.AddColumn(header,[|cell|])
355+
t.AddColumn(CompositeHeader.Output(IOType.Sample),[|CompositeCell.createFreeText "Sample"|])
356+
357+
let processes = t.GetProcesses()
358+
Expect.equal 1 processes.Length "Should have 1 process"
359+
let p = processes.[0]
360+
Expect.equal 1 p.ParameterValues.Value.Length "Should have 1 parameter value"
361+
let pv = p.ParameterValues.Value.[0]
362+
Expect.isNone pv.Value "Should have no value"
363+
let resultCategory = Expect.wantSome pv.Category "Should have category"
364+
Expect.equal resultCategory (ProtocolParameter.create(ParameterName = oa_temperature)) "Category should match"
365+
Expect.isNone pv.Unit "Should have no unit"
366+
367+
let t' = ArcTable.fromProcesses tableName1 processes
368+
Expect.arcTableEqual t' t "Table should be equal"
369+
)
370+
testCase "ParamValueUnitizedNoUnit" (fun () ->
371+
let t = ArcTable.init tableName1
372+
let header = CompositeHeader.Parameter oa_temperature
373+
let cell = CompositeCell.createUnitized ("5")
374+
t.AddColumn(CompositeHeader.Input(IOType.Source),[|CompositeCell.createFreeText "Source"|])
375+
t.AddColumn(header,[|cell|])
376+
t.AddColumn(CompositeHeader.Output(IOType.Sample),[|CompositeCell.createFreeText "Sample"|])
377+
378+
let processes = t.GetProcesses()
379+
Expect.equal 1 processes.Length "Should have 1 process"
380+
let p = processes.[0]
381+
Expect.equal 1 p.ParameterValues.Value.Length "Should have 1 parameter value"
382+
let pv = p.ParameterValues.Value.[0]
383+
let resultValue = Expect.wantSome pv.Value "Should have value"
384+
Expect.equal resultValue (Value.Int 5) "Value should match"
385+
let resultCategory = Expect.wantSome pv.Category "Should have category"
386+
Expect.equal resultCategory (ProtocolParameter.create(ParameterName = oa_temperature)) "Category should match"
387+
Expect.isNone pv.Unit "Should have no unit"
388+
389+
let t' = ArcTable.fromProcesses tableName1 processes
390+
Expect.arcTableEqual t' t "Table should be equal"
391+
)
392+
testCase "ParamValueTermEmpty" (fun () ->
393+
let t = ArcTable.init tableName1
394+
let header = CompositeHeader.Parameter oa_species
395+
let cell = CompositeCell.createTerm (OntologyAnnotation.create())
396+
t.AddColumn(CompositeHeader.Input(IOType.Source),[|CompositeCell.createFreeText "Source"|])
397+
t.AddColumn(header,[|cell|])
398+
t.AddColumn(CompositeHeader.Output(IOType.Sample),[|CompositeCell.createFreeText "Sample"|])
399+
400+
let processes = t.GetProcesses()
401+
Expect.equal 1 processes.Length "Should have 1 process"
402+
let p = processes.[0]
403+
Expect.equal 1 p.ParameterValues.Value.Length "Should have 1 parameter value"
404+
let pv = p.ParameterValues.Value.[0]
405+
Expect.isNone pv.Value "Should have no value"
406+
let resultCategory = Expect.wantSome pv.Category "Should have category"
407+
Expect.equal resultCategory (ProtocolParameter.create(ParameterName = oa_species)) "Category should match"
408+
Expect.isNone pv.Unit "Should have no unit"
409+
410+
let t' = ArcTable.fromProcesses tableName1 processes
411+
Expect.arcTableEqual t' t "Table should be equal"
412+
)
305413
]
306414

307415
let private tests_ArcTablesProcessSeq =

‎tests/Json/ARCtrl.Json.Tests.fsproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
<Compile Include="Assay.Tests.fs" />
2727
<Compile Include="Study.Tests.fs" />
2828
<Compile Include="Investigation.Tests.fs" />
29+
<Compile Include="Process\ProcessParameterValue.fs" />
2930
<Compile Include="Process\ProcessInput.Tests.fs" />
3031
<Compile Include="Process\Protocol.Tests.fs" />
3132
<Compile Include="Process\Process.Tests.fs" />

‎tests/Json/Main.fs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ let all = testSequenced <| testList "Json" [
1616
Tests.Study.main
1717
Tests.Investigation.main
1818
Tests.Template.main
19+
Tests.Process.ProcessParameterValue.main
1920
Tests.Process.ProcessInput.main
2021
Tests.Process.Protocol.main
2122
Tests.Process.Process.main

0 commit comments

Comments
 (0)