refactor return values of assay and study xlsx file readers

Author: HLWeil

src/ISADotNet.XLSX/AssayFile/AnnotationTable.fs

@@ -7,7 +7,7 @@ open ISADotNet
 module AnnotationTable = 
 
     /// Returns the protocol described by the headers and a function for parsing the values of the matrix to the processes of this protocol
-    let getProcessGetter protocolMetaData (nodes : seq<seq<string>>) =
+    let getProcessGetter (processNameRoot : string) (nodes : seq<seq<string>>) =
 
         let valueNodes =
             nodes
@@ -27,6 +27,19 @@ module AnnotationTable =
             |> Seq.fold (fun (pl,pvl) (p,pv) -> p.Value :: pl, pv :: pvl) ([],[])
             |> fun (l1,l2) -> List.rev l1, List.rev l2
 
+        let componentGetters =
+            valueNodes 
+            |> Seq.choose (fun (i,n) -> AnnotationNode.tryGetComponentGetter i n)
+            |> Seq.toList
+
+        let protocolTypeGetter = 
+            valueNodes 
+            |> Seq.tryPick (fun (i,n) -> AnnotationNode.tryGetProtocolTypeGetter i n)
+
+        let protocolREFGetter = 
+            valueNodes 
+            |> Seq.tryPick (fun (i,n) -> AnnotationNode.tryGetProtocolREFGetter i n)
+
         let dataFileGetter = nodes |> Seq.tryPick AnnotationNode.tryGetDataFileGetter
 
         let inputGetter,outputGetter =
@@ -113,15 +126,27 @@ module AnnotationTable =
                             [ProcessOutput.Sample sample]  
                 inputGetter,outputGetter
 
-        let protocol = {protocolMetaData with Parameters = Option.fromValueWithDefault [] parameters}
-    
-        characteristics,
-        factors,
-        protocol,
+        
+        
         fun (matrix : System.Collections.Generic.Dictionary<(int * string),string>) i ->
+
+            let pn = processNameRoot |> Option.fromValueWithDefault ""
+
+            let protocol : Protocol = 
+                Protocol.make 
+                    None
+                    (protocolREFGetter |> Option.mapOrDefault pn (fun f -> f matrix i))
+                    (protocolTypeGetter |> Option.map (fun f -> f matrix i))
+                    None
+                    None
+                    None
+                    (Option.fromValueWithDefault [] parameters)
+                    (componentGetters |> List.map (fun f -> f matrix i) |> Option.fromValueWithDefault [])
+                    None
+
             Process.make 
                 None 
-                None 
+                pn 
                 (Some protocol) 
                 (parameterValueGetters |> List.map (fun f -> f matrix i) |> Option.fromValueWithDefault [])
                 None
@@ -152,9 +177,12 @@ module AnnotationTable =
         |> Seq.collect (fun (protocol,processGroup) ->
             processGroup
             |> Seq.mapi (fun i p -> 
-                {p with Name =                         
-                        protocol.Value.Name |> Option.map (fun s -> sprintf "%s_%i" s i)
-                }
+                let name = 
+                    match p.Name with
+                    | Some n -> Some n
+                    | None | Some "" -> protocol |> Option.bind (fun p -> p.Name)
+                    |> Option.map (fun s -> sprintf "%s_%i" s i)
+                {p with Name = name}
             )
         )    
 

src/ISADotNet.XLSX/AssayFile/Assay.fs

@@ -22,11 +22,10 @@ module Process =
                     let j = kv.Key |> fst
                     if j > i  then i <- j
                 i + 1
-            let characteristic,factors,protocol,processGetter = 
+            let processGetter = 
                 AnnotationNode.splitIntoNodes matrixHeaders
-                |> AnnotationTable.getProcessGetter ({Protocol.empty with Name = Some processNameRoot}) 
-            characteristic,factors,protocol,
-            
+                |> AnnotationTable.getProcessGetter processNameRoot
+                     
             Seq.init len (processGetter sparseMatrix)
             |> AnnotationTable.mergeIdenticalProcesses
             |> AnnotationTable.indexRelatedProcessesByProtocolName
@@ -54,25 +53,22 @@ module Assay =
     /// sparseMatrix is a sparse representation of the sheet table, with the first part of the key being the column header and the second part being a zero based row index
     let fromSparseMatrix (processNameRoot:string) matrixHeaders (sparseMatrix : Dictionary<int*string,string>) = 
 
-        let characteristics,factors,protocols,processes = Process.fromSparseMatrix processNameRoot matrixHeaders sparseMatrix
-        factors,protocols,Assay.create(CharacteristicCategories = characteristics,ProcessSequence = Seq.toList processes)
+        let processes = Process.fromSparseMatrix processNameRoot matrixHeaders sparseMatrix
+        let characteristics = API.ProcessSequence.getCharacteristics processes
+        Assay.create(CharacteristicCategories = characteristics,ProcessSequence = Seq.toList processes)
 
 
     /// Returns an assay from a sequence of sparseMatrix representations of assay.xlsx sheets
     ///
     /// See "fromSparseMatrix" function for parameter documentation
     let fromSparseMatrices (sheets : (string*(string seq)*Dictionary<int*string,string>) seq) = 
-        let characteristics,factors,protocols,processes =
+        let processes =
             sheets
-            |> Seq.map (fun (name,matrixHeaders,matrix) -> Process.fromSparseMatrix name matrixHeaders matrix)
-            |> Seq.fold (fun (characteristics',factors',protocols',processes') (characteristics,factors,protocol,processes) ->
-                List.append characteristics' characteristics |> List.distinct,
-                List.append factors' factors |> List.distinct,
-                Seq.append protocols' (Seq.singleton protocol),
-                Seq.append processes' processes
-            ) (List.empty,List.empty,Seq.empty,Seq.empty)
+            |> Seq.collect (fun (name,matrixHeaders,matrix) -> Process.fromSparseMatrix name matrixHeaders matrix)
+            |> AnnotationTable.updateSamplesByThemselves
+            |> Seq.toList
 
-        let processes = AnnotationTable.updateSamplesByThemselves processes |> Seq.toList
+        let characteristics = API.ProcessSequence.getCharacteristics processes
 
         let assay = 
             match characteristics,processes with
@@ -81,7 +77,7 @@ module Assay =
             | cs,[] -> Assay.create(CharacteristicCategories = cs)
             | cs,ps -> Assay.create(CharacteristicCategories = cs,ProcessSequence = ps)
 
-        factors,protocols,assay
+        assay
 
 /// Diesen Block durch JS ersetzen ----> 
 
@@ -131,7 +127,7 @@ module Assay =
                 |> Sheet.Sheets.getSheets
                 |> Seq.map Sheet.getName
 
-            let factors,protocols,assay =
+            let assay =
                 sheetNames
                 |> Seq.collect (fun sheetName ->                    
                     match Spreadsheet.tryGetWorksheetPartBySheetName sheetName doc with
@@ -148,8 +144,6 @@ module Assay =
                 )
                 |> fromSparseMatrices // Feed the sheets (represented as sparse matrices) into the assay parser function
 
-            factors,
-            protocols |> Seq.toList,
             contacts,
             API.Update.UpdateByExisting.updateRecordType assayMetaData assay // Merges the assay containing the assay meta data and the assay containing the processes retrieved from the sheets
         with

src/ISADotNet.XLSX/StudyFile/Study.fs

@@ -25,25 +25,24 @@ module Study =
     ///
     /// sparseMatrix is a sparse representation of the sheet table, with the first part of the key being the column header and the second part being a zero based row index
     let fromSparseMatrix (processNameRoot:string) matrixHeaders (sparseMatrix : Dictionary<int*string,string>) = 
-        let characteristics,factors,protocol,processes = ISADotNet.XLSX.AssayFile.Process.fromSparseMatrix processNameRoot matrixHeaders sparseMatrix
-        Study.create(CharacteristicCategories = characteristics,Factors = factors, Protocols = [protocol], ProcessSequence = processes)
+        let processes = ISADotNet.XLSX.AssayFile.Process.fromSparseMatrix processNameRoot matrixHeaders sparseMatrix
+        let characteristics = API.ProcessSequence.getCharacteristics processes
+        let factors = API.ProcessSequence.getFactors processes
+        let protocols = API.ProcessSequence.getProtocols processes
+        Study.create(CharacteristicCategories = characteristics,Factors = factors, Protocols = protocols, ProcessSequence = processes)
 
     /// Returns a study from a sequence of sparseMatrix representations of study.xlsx sheets
     ///
     /// See "fromSparseMatrix" function for parameter documentation
     let fromSparseMatrices (sheets : (string*(string seq)*Dictionary<int*string,string>) seq) = 
-        let characteristics,factors,protocols,processes =
+        let processes =
             sheets
-            |> Seq.map (fun (name,matrixHeaders,matrix) -> ISADotNet.XLSX.AssayFile.Process.fromSparseMatrix name matrixHeaders matrix)
-            |> Seq.fold (fun (characteristics',factors',protocols',processes') (characteristics,factors,protocol,processes) ->
-                List.append characteristics' characteristics |> List.distinct,
-                List.append factors' factors |> List.distinct,
-                List.append protocols' (List.singleton protocol),
-                List.append processes' processes
-            ) (List.empty,List.empty,List.empty,List.empty)
-
-        let processes = ISADotNet.XLSX.AssayFile.AnnotationTable.updateSamplesByThemselves processes |> Seq.toList
-
+            |> Seq.collect (fun (name,matrixHeaders,matrix) -> ISADotNet.XLSX.AssayFile.Process.fromSparseMatrix name matrixHeaders matrix)
+            |> ISADotNet.XLSX.AssayFile.AnnotationTable.updateSamplesByThemselves 
+            |> Seq.toList
+        let characteristics = API.ProcessSequence.getCharacteristics processes
+        let factors = API.ProcessSequence.getFactors processes
+        let protocols = API.ProcessSequence.getProtocols processes
         Study.create(CharacteristicCategories = characteristics,Factors = factors, Protocols = protocols, ProcessSequence = processes)
 
 // Diesen Block durch JS ersetzen ----> 

src/ISADotnet/API/Assay.fs

@@ -208,4 +208,9 @@ module Assay =
     /// Returns the factors implemented by the processes contained in this assay
     let getFactors (assay : Assay) =
         assay.ProcessSequence
-        |> Option.map ProcessSequence.getFactors
+        |> Option.map ProcessSequence.getFactors
+
+    /// Returns the protocols implemented by the processes contained in this assay
+    let getProtocols (assay : Assay) =
+        assay.ProcessSequence
+        |> Option.map ProcessSequence.getProtocols

src/ISADotnet/API/Process.fs

@@ -108,13 +108,18 @@ module ProcessInput =
         | _ -> None
 
     /// If given process input contains characteristics, returns them
-    let tryGetCharacteristics (pi : ProcessInput) =
+    let tryGetCharacteristicValues (pi : ProcessInput) =
         match pi with
         | ProcessInput.Sample s     -> s.Characteristics
         | ProcessInput.Source s     -> s.Characteristics
         | ProcessInput.Material m   -> m.Characteristics
         | ProcessInput.Data _       -> None
 
+    /// If given process input contains characteristics, returns them
+    let tryGetCharacteristics (pi : ProcessInput) =
+        tryGetCharacteristicValues pi
+        |> Option.map (List.choose (fun c -> c.Category))
+
 /// Functions for handling the ProcessOutput Type
 module ProcessOutput =
 
@@ -174,12 +179,18 @@ module ProcessOutput =
 
 
     /// If given process output contains characteristics, returns them
-    let tryGetCharacteristics (po : ProcessOutput) =
+    let tryGetCharacteristicValues (po : ProcessOutput) =
         match po with
         | ProcessOutput.Sample s     -> s.Characteristics
         | ProcessOutput.Material m   -> m.Characteristics
         | ProcessOutput.Data _       -> None
 
+    /// If given process output contains characteristics, returns them
+    let tryGetCharacteristics (po : ProcessOutput) =
+        tryGetCharacteristicValues po
+        |> Option.map (List.choose (fun c -> c.Category))
+
+
     /// If given process output contains factors, returns them
     let tryGetFactorValues (po : ProcessOutput) =
         match po with
@@ -209,6 +220,29 @@ module Process =
             |> List.choose (fun pv -> pv.Category)
         | None -> []
 
+    /// Returns the characteristics describing the inputs of the process
+    let getInputCharacteristics (p: Process) =
+        match p.Inputs with
+        | Some ins ->
+            ins 
+            |> List.collect (fun inp -> ProcessInput.tryGetCharacteristics inp |> Option.defaultValue [])
+            |> List.distinct
+        | None -> []
+
+    /// Returns the characteristics describing the outputs of the process
+    let getOutputCharacteristics (p: Process) =
+        match p.Outputs with
+        | Some outs ->
+            outs 
+            |> List.collect (fun out -> ProcessOutput.tryGetCharacteristics out |> Option.defaultValue [])
+            |> List.distinct
+        | None -> []
+
+    /// Returns the characteristics describing the inputs and outputs of the process
+    let getCharacteristics (p: Process) =
+        getInputCharacteristics p @ getOutputCharacteristics p
+        |> List.distinct
+
     /// If the process implements the given parameter, return the list of input files together with their according parameter values of this parameter
     let tryGetInputsWithParameterBy (predicate : ProtocolParameter -> bool) (p : Process) =
         match p.ParameterValues with
@@ -232,21 +266,13 @@ module Process =
             | None -> None
         | None -> None
 
-    /// Returns the characteristics of the samples of the process
-    let getCharacteristics (p : Process) =
-        let materialAttributesOfValues (mvs : (MaterialAttributeValue list) Option) = 
-            mvs |> Option.defaultValue [] |> List.choose (fun mv -> mv.Category)
-        p.Inputs |> Option.defaultValue [] |> List.collect (ProcessInput.tryGetCharacteristics >> materialAttributesOfValues)
-        |> List.append (p.Outputs |> Option.defaultValue [] |> List.collect (ProcessOutput.tryGetCharacteristics >> materialAttributesOfValues))
-        |> List.distinct
-
     /// If the process implements the given characteristic, return the list of input files together with their according characteristic values of this characteristic
     let tryGetInputsWithCharacteristicBy (predicate : MaterialAttribute -> bool) (p : Process) =
         match p.Inputs with
         | Some is ->
             is
             |> List.choose (fun i ->
-                ProcessInput.tryGetCharacteristics i
+                ProcessInput.tryGetCharacteristicValues i
                 |> Option.defaultValue []
                 |> List.tryPick (fun mv -> 
                     match mv.Category with
@@ -264,7 +290,7 @@ module Process =
         | Some is,Some os ->
             List.zip is os
             |> List.choose (fun (i,o) ->
-                ProcessInput.tryGetCharacteristics i
+                ProcessInput.tryGetCharacteristicValues i
                 |> Option.defaultValue []
                 |> List.tryPick (fun mv -> 
                     match mv.Category with
@@ -320,6 +346,12 @@ module ProcessSequence =
             )        
         )
         |> List.distinct
+        
+    /// Returns the protocols the given processes impelement
+    let getProtocols (processSequence : Process list) =
+        processSequence
+        |> List.choose (fun p -> p.ExecutesProtocol)
+        |> List.distinct
 
     /// Returns a list of the processes, containing only the ones executing a protocol for which the predicate returns true
     let filterByProtocolBy (predicate : Protocol -> bool) (processSequence : Process list) =
@@ -352,6 +384,12 @@ module ProcessSequence =
         |> List.collect Process.getParameters
         |> List.distinct
 
+    /// Returns the characteristics describing the inputs and outputs of the processssequence
+    let getCharacteristics  (processSequence : Process list) =
+        processSequence
+        |> List.collect Process.getCharacteristics
+        |> List.distinct
+
     /// If the processes contain a process implementing the given parameter, return the list of input files together with their according parameter values of this parameter
     let getInputsWithCharacteristicBy (predicate:MaterialAttribute -> bool) (processSequence : Process list) =
         processSequence