add logging interface

Project.toml

@@ -1,7 +1,7 @@
 name = "Lighthouse"
 uuid = "ac2c24cd-07f0-4848-96b2-1b82c3ea0e59"
 authors = ["Beacon Biosignals, Inc."]
-version = "0.14.5"
+version = "0.14.6"
 
 [deps]
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"

docs/src/index.md

@@ -19,18 +19,42 @@ Lighthouse.is_early_stopping_exception
 ## The `learn!` Interface
 
 ```@docs
-LearnLogger
 learn!
-upon
 evaluate!
 predict!
-Lighthouse.forward_logs
-Lighthouse.log_evaluation_row!
 Lighthouse._calculate_ea_kappas
 Lighthouse._calculate_ira_kappas
 Lighthouse._calculate_spearman_correlation
 ```
 
+## The logging interface
+
+The following "primitives" must be defined for a logger to be used with Lighthouse:
+
+```@docs
+log_event!
+log_value!
+log_line_series!
+log_plot!
+step_logger!
+```
+
+These primitives can be used in implementations of [`train!`](@ref), [`evaluate!`](@ref), and [`predict!`](@ref), as well as in:
+
+```@docs
+Lighthouse.log_evaluation_row!
+```
+
+### `LearnLogger`s
+
+`LearnLoggers` are a Tensorboard-backed logger which comply with the above logging interface. They also support additional callback functionality with `upon`:
+
+```@docs
+LearnLogger
+upon
+Lighthouse.forward_logs
+```
+
 ## Performance Metrics
 
 ```@docs

src/Lighthouse.jl

@@ -25,5 +25,6 @@ export AbstractClassifier
 
 include("learn.jl")
 export LearnLogger, learn!, upon, evaluate!, predict!
+export log_event!, log_line_series!, log_plot!, step_logger!, log_value!
 
 end # module

src/classifier.jl

@@ -43,7 +43,7 @@ This method must be implemented for each `AbstractClassifier` subtype.
 function classes end
 
 """
-    Lighthouse.train!(classifier::AbstractClassifier, batches, logger::LearnLogger)
+    Lighthouse.train!(classifier::AbstractClassifier, batches, logger)
 
 Train `classifier` on the iterable `batches` for a single epoch. This function
 is called once per epoch by [`learn!`](@ref).

src/learn.jl

@@ -24,19 +24,45 @@ function LearnLogger(path, run_name; kwargs...)
     return LearnLogger(path, tensorboard_logger, Dict{String,Any}())
 end
 
+"""
+    log_event!(logger, value::AbstractString)
+
+Logs a string event given by `value` to `logger`.
+"""
+log_event!(logger, value)
+
 function log_event!(logger::LearnLogger, value)
     logged = string(now(), " | ", value)
     TensorBoardLogger.log_text(logger.tensorboard_logger, "events", logged)
     return logged
 end
 
+"""
+    log_plot!(logger, field::AbstractString, plot, plot_data)
+
+Log a `plot` to `logger` under field `field`.
+
+* `plot`: the plot itself
+* `plot_data`: an unstructured dictionary of values used in creating `plot`.
+
+See also [`log_line_series!`](@ref).
+"""
+log_plot!(logger, field::AbstractString, plot, plot_data)
+
 function log_plot!(logger::LearnLogger, field::AbstractString, plot, plot_data)
     values = get!(() -> Any[], logger.logged, field)
     push!(values, plot_data)
     TensorBoardLogger.log_image(logger.tensorboard_logger, field, plot; step=length(values))
     return plot
 end
 
+"""
+    log_plot!(logger, field::AbstractString, value)
+
+Log a scalar value `value` to `field`.
+"""
+log_value!(logger, field::AbstractString, value)
+
 function log_value!(logger::LearnLogger, field::AbstractString, value)
     values = get!(() -> Any[], logger.logged, field)
     push!(values, value)
@@ -45,12 +71,29 @@ function log_value!(logger::LearnLogger, field::AbstractString, value)
     return value
 end
 
+"""
+    log_line_series!(logger, field::AbstractString, curves, labels=1:length(curves))
+
+Logs a series plot to `logger` under `field`, where...
+
+- `curves` is an iterable of the form `Tuple{Vector{Real},Vector{Real}}`, where each tuple contains `(x-values, y-values)`, as in the `Lighthouse.EvaluationRow` field `per_class_roc_curves`
+- `labels` is the class label for each curve, which defaults to the numeric index of each curve.
+"""
+log_line_series!(logger, field::AbstractString, curves, labels=1:length(curves))
 
-function log_line_series!(logger::LearnLogger, field::AbstractString, series, series_labels)
-    @warn "`log_line_series!` not implemented for `LearnLogger`"
+function log_line_series!(logger::LearnLogger, field::AbstractString, curves, labels=1:length(curves))
+    @warn "`log_line_series!` not implemented for `LearnLogger`" maxlog=1
     return nothing
 end
 
+"""
+    step_logger!(logger)
+
+Increments the `logger`'s `step`, if any. Defaults to doing nothing.
+"""
+step_logger!(logger) = nothing
+
+
 """
     log_evaluation_row!(logger, field::AbstractString, metrics)
 
@@ -186,7 +229,7 @@ end
     evaluate!(predicted_hard_labels::AbstractVector,
               predicted_soft_labels::AbstractMatrix,
               elected_hard_labels::AbstractVector,
-              classes, logger::LearnLogger;
+              classes, logger;
               logger_prefix, logger_suffix,
               votes::Union{Nothing,AbstractMatrix}=nothing,
               thresholds=0.0:0.01:1.0,
@@ -842,6 +885,7 @@ end
 #####
 
 """
+    upon(logger::LearnLogger, field::AbstractString; condition, initial)
     upon(logged::Dict{String,Any}, field::AbstractString; condition, initial)
 
 Return a closure that can be called to check the most recent state of