Skip to content

Commit c2de6fc

Browse files
AlizterAlizter
committed
fix(pkg): resolve symlinks from (install) action
Signed-off-by: Ali Caglayan <[email protected]>
1 parent fab3dcc commit c2de6fc

File tree

10 files changed

+504
-4
lines changed

10 files changed

+504
-4
lines changed

‎src/dune_rules/pkg_rules.ml‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1924,6 +1924,38 @@ module Install_action = struct
19241924
Some (entry.section, dst)
19251925
;;
19261926

1927+
let rec resolve_symlinks_in dir =
1928+
match Readdir.read_directory_with_kinds dir with
1929+
| Error e -> Unix_error.Detailed.raise e
1930+
| Ok entries ->
1931+
List.iter entries ~f:(fun (fname, kind) ->
1932+
let path = Filename.concat dir fname in
1933+
match (kind : Unix.file_kind) with
1934+
| S_DIR -> resolve_symlinks_in path
1935+
| S_LNK ->
1936+
(match Fpath.follow_symlink path with
1937+
| Error (Unix_error e) -> Unix_error.Detailed.raise e
1938+
| Error Not_a_symlink ->
1939+
Code_error.raise
1940+
"resolve_symlinks_in: not a symlink"
1941+
[ "path", Dyn.string path ]
1942+
| Error Max_depth_exceeded ->
1943+
User_error.raise
1944+
[ Pp.textf
1945+
"Unable to resolve symlink %s: too many levels of symbolic links"
1946+
path
1947+
]
1948+
| Ok resolved ->
1949+
(match Unix.lstat resolved with
1950+
| { Unix.st_kind = S_REG; _ } ->
1951+
Fpath.unlink_exn path;
1952+
Io.portable_hardlink
1953+
~src:(Path.of_string resolved)
1954+
~dst:(Path.of_string path)
1955+
| _ -> ()))
1956+
| _ -> ())
1957+
;;
1958+
19271959
let action
19281960
{ package
19291961
; install_file
@@ -2011,6 +2043,14 @@ module Install_action = struct
20112043
let+ variables = Async.async (fun () -> read_variables config_file) in
20122044
{ Install_cookie.Gen.files; variables }
20132045
in
2046+
(* Resolve symlinks in target_dir so that the cache can store them. The
2047+
dune cache doesn't support symlinks, so we replace them with hardlinks
2048+
to their targets. *)
2049+
let* () =
2050+
Async.async (fun () ->
2051+
if Path.Untracked.exists (Path.build target_dir)
2052+
then resolve_symlinks_in (Path.Build.to_string target_dir))
2053+
in
20142054
(* Produce the cookie file in the standard path *)
20152055
let cookie_file = Path.build @@ Paths.install_cookie' target_dir in
20162056
Async.async (fun () ->

‎test/blackbox-tests/test-cases/pkg/install-symlinks/absolute-symlink-inside.t‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
Test symlinks using absolute paths that point inside the target directory.
2+
Since we use realpath, these resolve correctly and work with caching.
3+
4+
$ export DUNE_CACHE=enabled
5+
$ export DUNE_CACHE_ROOT=$PWD/_cache
6+
7+
$ make_lockdir
8+
9+
$ make_lockpkg foo <<EOF
10+
> (version 0.0.1)
11+
> (build
12+
> (progn
13+
> (run true BUILDING_FOO_PACKAGE)
14+
> (run mkdir -p %{lib}/%{pkg-self:name})
15+
> (run sh -c "echo 'real content' > %{lib}/%{pkg-self:name}/real.txt")
16+
> (run sh -c "ln -s \$(realpath %{lib}/%{pkg-self:name}/real.txt) %{lib}/%{pkg-self:name}/link.txt")
17+
> (run touch %{lib}/%{pkg-self:name}/META)))
18+
> EOF
19+
20+
$ cat > dune-project <<EOF
21+
> (lang dune 3.22)
22+
> (package
23+
> (name x)
24+
> (allow_empty)
25+
> (depends foo))
26+
> EOF
27+
28+
$ build_pkg foo
29+
30+
Verify the build ran:
31+
32+
$ count_trace BUILDING_FOO_PACKAGE
33+
1
34+
35+
The symlink is resolved to a regular file:
36+
37+
$ dune_cmd stat kind $(get_build_pkg_dir foo)/target/lib/foo/link.txt
38+
regular file
39+
40+
Both files are hardlinked:
41+
42+
$ dune_cmd stat hardlinks $(get_build_pkg_dir foo)/target/lib/foo/real.txt
43+
3
44+
$ dune_cmd stat hardlinks $(get_build_pkg_dir foo)/target/lib/foo/link.txt
45+
3
46+
47+
Clean and rebuild to verify cache restore:
48+
49+
$ rm -rf _build
50+
$ build_pkg foo
51+
52+
$ count_trace BUILDING_FOO_PACKAGE
53+
0

‎test/blackbox-tests/test-cases/pkg/install-symlinks/basic-caching.t‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -49,15 +49,15 @@ Check that the files exist:
4949
The symlink has been resolved to a regular file (hardlink to the target):
5050

5151
$ dune_cmd stat kind $(get_build_pkg_dir foo)/target/lib/foo/link.txt
52-
symbolic link
52+
regular file
5353

5454
Check hardlink count. Files should have hardlinks > 1 indicating they are cached.
5555
real.txt has 3 hardlinks: original, link.txt (resolved), and cache entry.
5656

5757
$ dune_cmd stat hardlinks $(get_build_pkg_dir foo)/target/lib/foo/real.txt
58-
1
58+
3
5959
$ dune_cmd stat hardlinks $(get_build_pkg_dir foo)/target/lib/foo/META
60-
1
60+
2
6161

6262
Clean and rebuild to verify cache restore:
6363

@@ -67,5 +67,5 @@ Clean and rebuild to verify cache restore:
6767
The build command should not be rerun since it will be restored from cache.
6868

6969
$ count_trace BUILDING_FOO_PACKAGE
70-
1
70+
0
7171

‎test/blackbox-tests/test-cases/pkg/install-symlinks/broken-symlink.t‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
Test that broken symlinks are detected during symlink resolution.
2+
3+
$ make_lockdir
4+
5+
$ make_lockpkg foo <<EOF
6+
> (version 0.0.1)
7+
> (build
8+
> (progn
9+
> (run mkdir -p %{lib}/%{pkg-self:name})
10+
> (run ln -s nonexistent.txt %{lib}/%{pkg-self:name}/broken.txt)
11+
> (run touch %{lib}/%{pkg-self:name}/META)))
12+
> EOF
13+
14+
$ cat > dune-project <<EOF
15+
> (lang dune 3.22)
16+
> (package
17+
> (name x)
18+
> (allow_empty)
19+
> (depends foo))
20+
> EOF
21+
22+
The broken symlink is detected:
23+
24+
$ build_pkg foo 2>&1 | sanitize_pkg_digest foo.0.0.1 \
25+
> | dune_cmd subst '\.sandbox/[a-f0-9]+' '.sandbox/$SANDBOX'
26+
Error:
27+
readlink(_build/.sandbox/$SANDBOX/_private/default/.pkg/foo.0.0.1-DIGEST_HASH/target/lib/foo/nonexistent.txt): No such file or directory
28+
-> required by
29+
_build/_private/default/.pkg/foo.0.0.1-DIGEST_HASH/target
30+
[1]

‎test/blackbox-tests/test-cases/pkg/install-symlinks/chain-through-outside.t‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
Test behavior when a symlink chain goes through a file outside the target directory.
2+
e.g., link.txt -> $PWD/intermediate.txt -> real.txt (where intermediate is outside)
3+
4+
The chain goes through an outside path, but the final target is inside target_dir.
5+
This case is pathological, but we want to exercise our symlink resolution code.
6+
7+
$ export DUNE_CACHE=enabled
8+
$ export DUNE_CACHE_ROOT=$PWD/_cache
9+
10+
$ make_lockdir
11+
12+
Create a directory outside the build for the intermediate symlink:
13+
14+
$ mkdir outside_dir
15+
16+
$ make_lockpkg foo <<EOF
17+
> (version 0.0.1)
18+
> (build
19+
> (progn
20+
> (run mkdir -p %{lib}/%{pkg-self:name})
21+
> (run sh -c "echo 'real content' > %{lib}/%{pkg-self:name}/real.txt")
22+
> (run sh -c "ln -sf \$(realpath %{lib}/%{pkg-self:name}/real.txt) $PWD/outside_dir/intermediate.txt")
23+
> (run ln -s $PWD/outside_dir/intermediate.txt %{lib}/%{pkg-self:name}/link.txt)
24+
> (run touch %{lib}/%{pkg-self:name}/META)))
25+
> EOF
26+
27+
$ cat > dune-project <<EOF
28+
> (lang dune 3.22)
29+
> (package
30+
> (name x)
31+
> (allow_empty)
32+
> (depends foo))
33+
> EOF
34+
35+
Build the package:
36+
37+
$ build_pkg foo
38+
39+
The symlink should be resolved to a regular file:
40+
41+
$ dune_cmd stat kind $(get_build_pkg_dir foo)/target/lib/foo/link.txt
42+
regular file
43+
44+
link.txt and real.txt should be hardlinks to the same file (hardlink count > 1):
45+
46+
$ dune_cmd stat hardlinks $(get_build_pkg_dir foo)/target/lib/foo/link.txt
47+
3
48+
$ dune_cmd stat hardlinks $(get_build_pkg_dir foo)/target/lib/foo/real.txt
49+
3
50+
51+
The outside intermediate should not have any hardlinks (still just a symlink):
52+
53+
$ dune_cmd stat hardlinks outside_dir/intermediate.txt
54+
1
55+
56+
$ cat $(get_build_pkg_dir foo)/target/lib/foo/link.txt
57+
real content
58+
59+
Caching works - the symlink was resolved so it can be cached:
60+
61+
$ rm -rf _build
62+
$ build_pkg foo
63+
$ dune_cmd stat hardlinks $(get_build_pkg_dir foo)/target/lib/foo/link.txt
64+
3

‎test/blackbox-tests/test-cases/pkg/install-symlinks/dir-symlink.t‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
Test behavior with directory symlinks.
2+
3+
$ make_lockdir
4+
5+
--------------------------------------------------------------------------------
6+
7+
Case 1: No .install file, directory symlink in %{lib}, engine rejects it.
8+
9+
$ make_lockpkg foo <<EOF
10+
> (version 0.0.1)
11+
> (build
12+
> (progn
13+
> (run mkdir -p %{lib}/%{pkg-self:name})
14+
> (run sh -c "echo 'content' > %{lib}/%{pkg-self:name}/file.txt")
15+
> (run mkdir -p %{lib}/%{pkg-self:name}/real_subdir)
16+
> (run ln -s real_subdir %{lib}/%{pkg-self:name}/subdir_link)
17+
> (run touch %{lib}/%{pkg-self:name}/META)))
18+
> EOF
19+
20+
CR-someday Alizter: Maybe in this case it actually makes sense to resolve the directory symlink somehow.
21+
22+
$ build_pkg foo 2>&1 | sanitize_pkg_digest foo.0.0.1
23+
Error: Error trying to read targets after a rule was run:
24+
- default/.pkg/foo.0.0.1-DIGEST_HASH/target/lib/foo/subdir_link: Unexpected file kind "S_DIR" (directory)
25+
-> required by
26+
_build/_private/default/.pkg/foo.0.0.1-DIGEST_HASH/target
27+
[1]
28+
29+
--------------------------------------------------------------------------------
30+
31+
Case 2: .install file explicitly includes directory symlink.
32+
33+
CR-someday Alizter: What should happen in this case? Seems we didn't catch the
34+
error correctly. Engine bug? Let's try to repro later.
35+
36+
$ make_lockpkg foo <<EOF
37+
> (version 0.0.1)
38+
> (build
39+
> (progn
40+
> (run mkdir -p real_subdir)
41+
> (run sh -c "echo 'content' > real_subdir/file.txt")
42+
> (run ln -s real_subdir subdir_link)
43+
> (run sh -c "echo 'lib: [\"real_subdir/file.txt\" \"subdir_link\"]' > foo.install")))
44+
> EOF
45+
46+
$ build_pkg foo 2>&1 | sanitize_pkg_digest foo.0.0.1 | sed -E 's#\.sandbox/[^/]+#.sandbox/SANDBOX#g'
47+
Error:
48+
link(_build/.sandbox/SANDBOX/_private/default/.pkg/foo.0.0.1-DIGEST_HASH/target/lib/foo/subdir_link): Operation not permitted
49+
-> required by
50+
_build/_private/default/.pkg/foo.0.0.1-DIGEST_HASH/target
51+
[1]
52+
53+
-------------------------------------------------------------------------------
54+
55+
Case 3: .install file excludes the directory symlink.
56+
57+
$ make_lockpkg foo <<EOF
58+
> (version 0.0.1)
59+
> (build
60+
> (progn
61+
> (run mkdir -p real_subdir)
62+
> (run sh -c "echo 'content' > real_subdir/file.txt")
63+
> (run ln -s file.txt real_subdir/link.txt)
64+
> (run ln -s real_subdir subdir_link)
65+
> (run sh -c "echo 'lib: [\"real_subdir/file.txt\" \"real_subdir/link.txt\"]' > foo.install")))
66+
> EOF
67+
68+
$ build_pkg foo
69+
70+
The directory symlink is not in the installed targets, so the engine does not
71+
reject it. Only the real directory contents are installed:
72+
73+
$ ls $(get_build_pkg_dir foo)/target/lib/foo
74+
file.txt
75+
link.txt
76+
77+
The file symlink is converted to a hardlink:
78+
79+
$ dune_cmd stat kind $(get_build_pkg_dir foo)/target/lib/foo/link.txt
80+
regular file
81+
82+
$ dune_cmd stat hardlinks $(get_build_pkg_dir foo)/target/lib/foo/link.txt
83+
2

‎test/blackbox-tests/test-cases/pkg/install-symlinks/link-permission-error.t‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
Test that link errors cause the build to fail.
2+
3+
If the directory containing the symlink is made non-writable, unlinking the
4+
symlink before hardlinking will fail with EACCES.
5+
6+
$ make_lockdir
7+
8+
$ make_lockpkg foo <<EOF
9+
> (version 0.0.1)
10+
> (build
11+
> (progn
12+
> (run mkdir -p %{lib}/%{pkg-self:name})
13+
> (run sh -c "echo 'real content' > %{lib}/%{pkg-self:name}/real.txt")
14+
> (run ln -s real.txt %{lib}/%{pkg-self:name}/link.txt)
15+
> (run touch %{lib}/%{pkg-self:name}/META)
16+
> (run chmod -w %{lib}/%{pkg-self:name})))
17+
> EOF
18+
19+
$ cat > dune-project <<EOF
20+
> (lang dune 3.22)
21+
> (package
22+
> (name x)
23+
> (allow_empty)
24+
> (depends foo))
25+
> EOF
26+
27+
Build fails with a symlink resolution error:
28+
29+
$ build_pkg foo 2>&1 | sanitize_pkg_digest foo.0.0.1 | dune_cmd subst '\.sandbox/[^/]+' '.sandbox/SANDBOX'
30+
Error: failed to delete sandbox in
31+
_build/.sandbox/SANDBOX
32+
Reason:
33+
rmdir(_build/.sandbox/SANDBOX/_private/default/.pkg/foo.0.0.1-DIGEST_HASH/target/lib/foo): Directory not empty
34+
-> required by
35+
_build/_private/default/.pkg/foo.0.0.1-DIGEST_HASH/target
36+
Error:
37+
unlink(_build/.sandbox/SANDBOX/_private/default/.pkg/foo.0.0.1-DIGEST_HASH/target/lib/foo/link.txt): Permission denied
38+
-> required by
39+
_build/_private/default/.pkg/foo.0.0.1-DIGEST_HASH/target
40+
[1]
41+
42+
Restore permissions for cleanup:
43+
44+
$ chmod -R +w _build 2>/dev/null; rm -rf _build

0 commit comments

Comments
 (0)