pidlock cache file precompilation (#49052)

Author: IanButterworth

base/loading.jl

@@ -1902,8 +1902,17 @@ function _require(pkg::PkgId, env=nothing)
                     @goto load_from_cache
                 end
                 # spawn off a new incremental pre-compile task for recursive `require` calls
-                cachefile = compilecache(pkg, path)
-                if isa(cachefile, Exception)
+                cachefile_or_module = maybe_cachefile_lock(pkg, path) do
+                    # double-check now that we have lock
+                    m = _require_search_from_serialized(pkg, path, UInt128(0))
+                    m isa Module && return m
+                    compilecache(pkg, path)
+                end
+                cachefile_or_module isa Module && return cachefile_or_module::Module
+                cachefile = cachefile_or_module
+                if isnothing(cachefile) # maybe_cachefile_lock returns nothing if it had to wait for another process
+                    @goto load_from_cache # the new cachefile will have the newest mtime so will come first in the search
+                elseif isa(cachefile, Exception)
                     if precompilableerror(cachefile)
                         verbosity = isinteractive() ? CoreLogging.Info : CoreLogging.Debug
                         @logmsg verbosity "Skipping precompilation since __precompile__(false). Importing $pkg."
@@ -2805,6 +2814,35 @@ function show(io::IO, cf::CacheFlags)
     print(io, ", opt_level = ", cf.opt_level)
 end
 
+# Set by FileWatching.__init__()
+global mkpidlock_hook
+global trymkpidlock_hook
+global parse_pidfile_hook
+
+# allows processes to wait if another process is precompiling a given source already
+function maybe_cachefile_lock(f, pkg::PkgId, srcpath::String)
+    if @isdefined(mkpidlock_hook) && @isdefined(trymkpidlock_hook) && @isdefined(parse_pidfile_hook)
+        pidfile = string(srcpath, ".pidlock")
+        cachefile = invokelatest(trymkpidlock_hook, f, pidfile)
+        if cachefile === false
+            pid, hostname, age = invokelatest(parse_pidfile_hook, pidfile)
+            verbosity = isinteractive() ? CoreLogging.Info : CoreLogging.Debug
+            if isempty(hostname) || hostname == gethostname()
+                @logmsg verbosity "Waiting for another process (pid: $pid) to finish precompiling $pkg"
+            else
+                @logmsg verbosity "Waiting for another machine (hostname: $hostname, pid: $pid) to finish precompiling $pkg"
+            end
+            # wait until the lock is available, but don't actually acquire it
+            # returning nothing indicates a process waited for another
+            return invokelatest(mkpidlock_hook, Returns(nothing), pidfile)
+        end
+        return cachefile
+    else
+        # for packages loaded before FileWatching.__init__()
+        f()
+    end
+end
+
 # returns true if it "cachefile.ji" is stale relative to "modpath.jl" and build_id for modkey
 # otherwise returns the list of dependencies to also check
 @constprop :none function stale_cachefile(modpath::String, cachefile::String; ignore_loaded::Bool = false)

stdlib/FileWatching/docs/src/index.md

@@ -20,6 +20,7 @@ A simple utility tool for creating advisory pidfiles (lock files).
 
 ```@docs
 mkpidlock
+trymkpidlock
 close(lock::LockMonitor)
 ```
 

stdlib/FileWatching/src/FileWatching.jl

@@ -18,7 +18,8 @@ export
     PollingFileWatcher,
     FDWatcher,
     # pidfile:
-    mkpidlock
+    mkpidlock,
+    trymkpidlock
 
 import Base: @handle_as, wait, close, eventloop, notify_error, IOError,
     _sizeof_uv_poll, _sizeof_uv_fs_poll, _sizeof_uv_fs_event, _uv_hook_close, uv_error, _UVError,
@@ -462,6 +463,11 @@ function __init__()
     global uv_jl_fspollcb = @cfunction(uv_fspollcb, Cvoid, (Ptr{Cvoid}, Cint, Ptr{Cvoid}, Ptr{Cvoid}))
     global uv_jl_fseventscb_file = @cfunction(uv_fseventscb_file, Cvoid, (Ptr{Cvoid}, Ptr{Int8}, Int32, Int32))
     global uv_jl_fseventscb_folder = @cfunction(uv_fseventscb_folder, Cvoid, (Ptr{Cvoid}, Ptr{Int8}, Int32, Int32))
+
+    Base.mkpidlock_hook = mkpidlock
+    Base.trymkpidlock_hook = trymkpidlock
+    Base.parse_pidfile_hook = Pidfile.parse_pidfile
+
     nothing
 end
 
@@ -885,6 +891,6 @@ function poll_file(s::AbstractString, interval_seconds::Real=5.007, timeout_s::R
 end
 
 include("pidfile.jl")
-import .Pidfile: mkpidlock
+import .Pidfile: mkpidlock, trymkpidlock
 
 end

stdlib/FileWatching/src/pidfile.jl

@@ -1,7 +1,7 @@
 module Pidfile
 
 
-export mkpidlock
+export mkpidlock, trymkpidlock
 
 using Base:
     IOError, UV_EEXIST, UV_ESRCH,
@@ -41,6 +41,16 @@ Optional keyword arguments:
 """
 function mkpidlock end
 
+"""
+    trymkpidlock([f::Function], at::String, [pid::Cint, proc::Process]; kwopts...)
+
+Like `mkpidlock` except returns `false` instead of waiting if the file is already locked.
+
+!!! compat "Julia 1.10"
+    This function requires at least Julia 1.10.
+
+"""
+function trymkpidlock end
 
 # mutable only because we want to add a finalizer
 mutable struct LockMonitor
@@ -95,6 +105,18 @@ function mkpidlock(at::String, proc::Process; kwopts...)
     return lock
 end
 
+function trymkpidlock(args...; kwargs...)
+    try
+        mkpidlock(args...; kwargs..., wait=false)
+    catch ex
+        if ex isa PidlockedError
+            return false
+        else
+            rethrow()
+        end
+    end
+end
+
 """
     Base.touch(::Pidfile.LockMonitor)
 
@@ -192,8 +214,12 @@ function tryopen_exclusive(path::String, mode::Integer = 0o444)
     return nothing
 end
 
+struct PidlockedError <: Exception
+    msg::AbstractString
+end
+
 """
-    open_exclusive(path::String; mode, poll_interval, stale_age) :: File
+    open_exclusive(path::String; mode, poll_interval, wait, stale_age) :: File
 
 Create a new a file for read-write advisory-exclusive access.
 If `wait` is `false` then error out if the lock files exist
@@ -218,7 +244,7 @@ function open_exclusive(path::String;
             file = tryopen_exclusive(path, mode)
         end
         if file === nothing
-            error("Failed to get pidfile lock for $(repr(path)).")
+            throw(PidlockedError("Failed to get pidfile lock for $(repr(path))."))
         else
             return file
         end

stdlib/FileWatching/test/pidfile.jl

@@ -180,14 +180,14 @@ end
         Base.errormonitor(rmtask)
 
         t1 = time()
-        @test_throws ErrorException open_exclusive("pidfile", wait=false)
+        @test_throws Pidfile.PidlockedError open_exclusive("pidfile", wait=false)
         @test time()-t1 ≈ 0 atol=1
 
         sleep(1)
         @test !deleted
 
         t1 = time()
-        @test_throws ErrorException open_exclusive("pidfile", wait=false)
+        @test_throws Pidfile.PidlockedError open_exclusive("pidfile", wait=false)
         @test time()-t1 ≈ 0 atol=1
 
         wait(rmtask)
@@ -246,7 +246,7 @@ end
     Base.errormonitor(waittask)
 
     # mkpidlock with no waiting
-    t = @elapsed @test_throws ErrorException mkpidlock("pidfile", wait=false)
+    t = @elapsed @test_throws Pidfile.PidlockedError mkpidlock("pidfile", wait=false)
     @test t ≈ 0 atol=1
 
     t = @elapsed lockf1 = mkpidlock(joinpath(dir, "pidfile"))
@@ -354,7 +354,7 @@ end
     @test lockf.update === nothing
 
     sleep(1)
-    t = @elapsed @test_throws ErrorException mkpidlock("pidfile-2", wait=false, stale_age=1, poll_interval=1, refresh=0)
+    t = @elapsed @test_throws Pidfile.PidlockedError mkpidlock("pidfile-2", wait=false, stale_age=1, poll_interval=1, refresh=0)
     @test t ≈ 0 atol=1
 
     sleep(5)