add logging interface

Project.toml

@@ -1,7 +1,7 @@
 name = "Lighthouse"
 uuid = "ac2c24cd-07f0-4848-96b2-1b82c3ea0e59"
 authors = ["Beacon Biosignals, Inc."]
-version = "0.14.5"
+version = "0.14.6"
 
 [deps]
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"

README.md

@@ -37,4 +37,6 @@ This will install Lighthouse to the default package development directory, `~/.j
 
 ### TensorBoard
 
-Note that Lighthouse logs metrics to a user-specified path in [TensorBoard's](https://github.com/tensorflow/tensorboard) `logdir` format. TensorBoard can be installed via `python3 -m pip install tensorboard` (note: if you have `tensorflow>=1.14`, you should already have `tensorboard`). Once TensorBoard is installed, you can view Lighthouse-generated metrics via `tensorboard --logdir path` where `path` is the path specified by `Lighthouse.LearnLogger`. From there, TensorBoard itself can be used/configured however you like; see https://github.com/tensorflow/tensorboard for more information.
+Note that Lighthouse's `LearnLogger` logs metrics to a user-specified path in [TensorBoard's](https://github.com/tensorflow/tensorboard) `logdir` format. TensorBoard can be installed via `python3 -m pip install tensorboard` (note: if you have `tensorflow>=1.14`, you should already have `tensorboard`). Once TensorBoard is installed, you can view Lighthouse-generated metrics via `tensorboard --logdir path` where `path` is the path specified by `Lighthouse.LearnLogger`. From there, TensorBoard itself can be used/configured however you like; see https://github.com/tensorflow/tensorboard for more information.
+
+You can use alternative loggers, as long as they comply with the [logging interface](https://beacon-biosignals.github.io/Lighthouse.jl/dev#The-logging-interface).

docs/src/index.md

@@ -19,18 +19,42 @@ Lighthouse.is_early_stopping_exception
 ## The `learn!` Interface
 
 ```@docs
-LearnLogger
 learn!
-upon
 evaluate!
 predict!
-Lighthouse.forward_logs
-Lighthouse.log_evaluation_row!
 Lighthouse._calculate_ea_kappas
 Lighthouse._calculate_ira_kappas
 Lighthouse._calculate_spearman_correlation
 ```
 
+## The logging interface
+
+The following "primitives" must be defined for a logger to be used with Lighthouse:
+
+```@docs
+log_value!
+log_line_series!
+log_plot!
+step_logger!
+```
+
+These primitives can be used in implementations of [`train!`](@ref), [`evaluate!`](@ref), and [`predict!`](@ref), as well as in:
+
+```@docs
+log_event!
+Lighthouse.log_evaluation_row!
+```
+
+### `LearnLogger`s
+
+`LearnLoggers` are a Tensorboard-backed logger which comply with the above logging interface. They also support additional callback functionality with `upon`:
+
+```@docs
+LearnLogger
+upon
+Lighthouse.forward_logs
+```
+
 ## Performance Metrics
 
 ```@docs

src/LearnLogger.jl

@@ -0,0 +1,95 @@
+#####
+##### `LearnLogger` implementation of logging interface
+#####
+
+"""
+    LearnLogger
+
+A struct that wraps a `TensorBoardLogger.TBLogger` in order to enforce the following:
+
+- all values logged to Tensorboard should be accessible to the `post_epoch_callback`
+  argument to [`learn!`](@ref)
+- all values that are cached during [`learn!`](@ref) should be logged to Tensorboard
+
+To access values logged to a `LearnLogger` instance, inspect the instance's `logged` field.
+"""
+struct LearnLogger
+    path::String
+    tensorboard_logger::TensorBoardLogger.TBLogger
+    logged::Dict{String,Vector{Any}}
+end
+
+function LearnLogger(path, run_name; kwargs...)
+    tensorboard_logger = TBLogger(joinpath(path, run_name); kwargs...)
+    return LearnLogger(path, tensorboard_logger, Dict{String,Any}())
+end
+
+function log_value!(logger::LearnLogger, field::AbstractString, value)
+    values = get!(() -> Any[], logger.logged, field)
+    push!(values, value)
+    TensorBoardLogger.log_value(logger.tensorboard_logger, field, value;
+                                step=length(values))
+    return value
+end
+
+function log_event!(logger::LearnLogger, value::AbstractString)
+    logged = string(now(), " | ", value)
+    TensorBoardLogger.log_text(logger.tensorboard_logger, "events", logged)
+    return logged
+end
+
+function log_plot!(logger::LearnLogger, field::AbstractString, plot, plot_data)
+    values = get!(() -> Any[], logger.logged, field)
+    push!(values, plot_data)
+    TensorBoardLogger.log_image(logger.tensorboard_logger, field, plot; step=length(values))
+    return plot
+end
+
+function log_line_series!(logger::LearnLogger, field::AbstractString, curves, labels=1:length(curves))
+    @warn "`log_line_series!` not implemented for `LearnLogger`" maxlog=1
+    return nothing
+end
+
+"""
+    Base.flush(logger::LearnLogger)
+
+Persist possibly transient logger state.
+"""
+Base.flush(logger::LearnLogger) = nothing
+
+"""
+    forwarding_task = forward_logs(channel, logger::LearnLogger)
+
+Forwards logs with values supported by `TensorBoardLogger` to `logger::LearnLogger`:
+- string events of type `AbstractString`
+- scalars of type `Union{Real,Complex}`
+- plots that `TensorBoardLogger` can convert to raster images
+
+returns the `forwarding_task:::Task` that does the forwarding.
+To cleanly stop forwarding, `close(channel)` and `wait(forwarding_task)`.
+
+outbox is a Channel or RemoteChannel of Pair{String, Any}
+field names starting with "__plot__" forward to TensorBoardLogger.log_image
+"""
+function forward_logs(outbox, logger::LearnLogger)
+    @async try
+        while true
+            (field, value) = take!(outbox)
+            if typeof(value) <: AbstractString
+                log_event!(logger, value)
+            elseif startswith(field, "__plot__")
+                original_field = field[9:end]
+                values = get!(() -> Any[], logger.logged, original_field)
+                TensorBoardLogger.log_image(logger.tensorboard_logger, original_field,
+                                            value; step=length(values))
+            elseif typeof(value) <: Union{Real,Complex}
+                log_value!(logger, field, value)
+            end
+        end
+    catch e
+        if !(isa(e, InvalidStateException) && e.state == :closed)
+            @error "error forwarding logs, STOPPING FORWARDING!" exception = (e,
+                                                                              catch_backtrace())
+        end
+    end
+end

src/Lighthouse.jl

@@ -23,7 +23,11 @@ export confusion_matrix, accuracy, binary_statistics, cohens_kappa, calibration_
 include("classifier.jl")
 export AbstractClassifier
 
+include("LearnLogger.jl")
+export LearnLogger
+
 include("learn.jl")
-export LearnLogger, learn!, upon, evaluate!, predict!
+export learn!, upon, evaluate!, predict!
+export log_event!, log_line_series!, log_plot!, step_logger!, log_value!
 
 end # module

src/classifier.jl

@@ -43,7 +43,7 @@ This method must be implemented for each `AbstractClassifier` subtype.
 function classes end
 
 """
-    Lighthouse.train!(classifier::AbstractClassifier, batches, logger::LearnLogger)
+    Lighthouse.train!(classifier::AbstractClassifier, batches, logger)
 
 Train `classifier` on the iterable `batches` for a single epoch. This function
 is called once per epoch by [`learn!`](@ref).

src/learn.jl

@@ -1,56 +1,60 @@
 #####
-##### `LearnLogger`
+##### Logging interface
 #####
 
+# These must be implemented by every logger type.
+
 """
-    LearnLogger
+    log_plot!(logger, field::AbstractString, plot, plot_data)
 
-A struct that wraps a `TensorBoardLogger.TBLogger` in order to enforce the following:
+Log a `plot` to `logger` under field `field`.
 
-- all values logged to Tensorboard should be accessible to the `post_epoch_callback`
-  argument to [`learn!`](@ref)
-- all values that are cached during [`learn!`](@ref) should be logged to Tensorboard
+* `plot`: the plot itself
+* `plot_data`: an unstructured dictionary of values used in creating `plot`.
 
-To access values logged to a `LearnLogger` instance, inspect the instance's `logged` field.
+See also [`log_line_series!`](@ref).
 """
-struct LearnLogger
-    path::String
-    tensorboard_logger::TensorBoardLogger.TBLogger
-    logged::Dict{String,Vector{Any}}
-end
+log_plot!(logger, field::AbstractString, plot, plot_data)
 
-function LearnLogger(path, run_name; kwargs...)
-    tensorboard_logger = TBLogger(joinpath(path, run_name); kwargs...)
-    return LearnLogger(path, tensorboard_logger, Dict{String,Any}())
-end
 
-function log_event!(logger::LearnLogger, value)
-    logged = string(now(), " | ", value)
-    TensorBoardLogger.log_text(logger.tensorboard_logger, "events", logged)
-    return logged
-end
+"""
+    log_value!(logger, field::AbstractString, value)
 
-function log_plot!(logger::LearnLogger, field::AbstractString, plot, plot_data)
-    values = get!(() -> Any[], logger.logged, field)
-    push!(values, plot_data)
-    TensorBoardLogger.log_image(logger.tensorboard_logger, field, plot; step=length(values))
-    return plot
-end
+Log a value `value` to `field`.
+"""
+log_value!(logger, field::AbstractString, value)
 
-function log_value!(logger::LearnLogger, field::AbstractString, value)
-    values = get!(() -> Any[], logger.logged, field)
-    push!(values, value)
-    TensorBoardLogger.log_value(logger.tensorboard_logger, field, value;
-                                step=length(values))
-    return value
-end
 
+"""
+    log_line_series!(logger, field::AbstractString, curves, labels=1:length(curves))
 
-function log_line_series!(logger::LearnLogger, field::AbstractString, series, series_labels)
-    @warn "`log_line_series!` not implemented for `LearnLogger`"
-    return nothing
+Logs a series plot to `logger` under `field`, where...
+
+- `curves` is an iterable of the form `Tuple{Vector{Real},Vector{Real}}`, where each tuple contains `(x-values, y-values)`, as in the `Lighthouse.EvaluationRow` field `per_class_roc_curves`
+- `labels` is the class label for each curve, which defaults to the numeric index of each curve.
+"""
+log_line_series!(logger, field::AbstractString, curves, labels=1:length(curves))
+
+# The following have default implementations.
+
+"""
+    step_logger!(logger)
+
+Increments the `logger`'s `step`, if any. Defaults to doing nothing.
+"""
+step_logger!(::Any) = nothing
+
+
+"""
+    log_event!(logger, value::AbstractString)
+
+Logs a string event given by `value` to `logger`. Defaults to calling `log_value!` with a field named `event`.
+"""
+function log_event!(logger, value::AbstractString)
+    return log_value!(logger, "event", string(now(), " | ", value))
 end
 
+
 """
     log_evaluation_row!(logger, field::AbstractString, metrics)
 
@@ -83,49 +87,6 @@ function log_resource_info!(f, logger, section::AbstractString; suffix::Abstract
     return result
 end
 
-"""
-    Base.flush(logger)
-
-Persist possibly transient logger state.
-"""
-Base.flush(logger::LearnLogger) = nothing
-
-"""
-    forwarding_task = forward_logs(channel, logger::LearnLogger)
-
-Forwards logs with values supported by `TensorBoardLogger` to `logger::LearnLogger`:
-- string events of type `AbstractString`
-- scalars of type `Union{Real,Complex}`
-- plots that `TensorBoardLogger` can convert to raster images
-
-returns the `forwarding_task:::Task` that does the forwarding.
-To cleanly stop forwarding, `close(channel)` and `wait(forwarding_task)`.
-
-outbox is a Channel or RemoteChannel of Pair{String, Any}
-field names starting with "__plot__" forward to TensorBoardLogger.log_image
-"""
-function forward_logs(outbox, logger::LearnLogger)
-    @async try
-        while true
-            (field, value) = take!(outbox)
-            if typeof(value) <: AbstractString
-                log_event!(logger, value)
-            elseif startswith(field, "__plot__")
-                original_field = field[9:end]
-                values = get!(() -> Any[], logger.logged, original_field)
-                TensorBoardLogger.log_image(logger.tensorboard_logger, original_field,
-                                            value; step=length(values))
-            elseif typeof(value) <: Union{Real,Complex}
-                log_value!(logger, field, value)
-            end
-        end
-    catch e
-        if !(isa(e, InvalidStateException) && e.state == :closed)
-            @error "error forwarding logs, STOPPING FORWARDING!" exception = (e,
-                                                                              catch_backtrace())
-        end
-    end
-end
 
 #####
 ##### `predict!!`
@@ -186,7 +147,7 @@ end
     evaluate!(predicted_hard_labels::AbstractVector,
               predicted_soft_labels::AbstractMatrix,
               elected_hard_labels::AbstractVector,
-              classes, logger::LearnLogger;
+              classes, logger;
               logger_prefix, logger_suffix,
               votes::Union{Nothing,AbstractMatrix}=nothing,
               thresholds=0.0:0.01:1.0,
@@ -842,6 +803,7 @@ end
 #####
 
 """
+    upon(logger::LearnLogger, field::AbstractString; condition, initial)
     upon(logged::Dict{String,Any}, field::AbstractString; condition, initial)
 
 Return a closure that can be called to check the most recent state of

test/logger.jl

@@ -30,5 +30,7 @@ end
     mktempdir() do logdir
         logger = LearnLogger(logdir, "test_run")
         @test isnothing(Lighthouse.log_line_series!(logger, "foo", 3, 2))
+
+        @test isnothing(step_logger!(logger))
     end
 end