From 8b6ecb2e903233742c42efb9399fa8c6f2b9c9f8 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 21 Sep 2022 19:13:10 +0800 Subject: [PATCH 001/113] print information about filtered ref-specs as well. Maybe this helps to understand how ls-refs extensions can be used. --- gitoxide-core/src/repository/remote.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/gitoxide-core/src/repository/remote.rs b/gitoxide-core/src/repository/remote.rs index 7325656cc31..81e4fbf8599 100644 --- a/gitoxide-core/src/repository/remote.rs +++ b/gitoxide-core/src/repository/remote.rs @@ -164,6 +164,15 @@ mod refs_impl { } } } + if map.remote_refs.len() - map.mappings.len() != 0 { + writeln!( + err, + "server sent {} tips, {} were filtered due to {} refspec(s).", + map.remote_refs.len(), + map.remote_refs.len() - map.mappings.len(), + refspecs.len() + )?; + } Ok(()) } From 6df179b5cf831402444cc78429a57f835358376e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 21 Sep 2022 21:51:47 +0800 Subject: [PATCH 002/113] feat: `RefSpecRef::prefix()` to return the two-component prefix of a refspec's source. #(450) --- git-refspec/src/spec.rs | 13 ++++++++++- git-refspec/tests/refspec.rs | 1 + git-refspec/tests/spec/mod.rs | 41 +++++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 git-refspec/tests/spec/mod.rs diff --git a/git-refspec/src/spec.rs b/git-refspec/src/spec.rs index a9654f02ad4..84657c90d13 100644 --- a/git-refspec/src/spec.rs +++ b/git-refspec/src/spec.rs @@ -1,4 +1,4 @@ -use bstr::BStr; +use bstr::{BStr, ByteSlice}; use crate::{ instruction::{Fetch, Push}, @@ -117,6 +117,17 @@ impl RefSpecRef<'_> { } } + /// Derive the prefix from the `source` side of this spec, if possible. + /// + /// This means it starts with `refs/`. Note that it won't contain more than two components, like `refs/heads/` + pub fn prefix(&self) -> Option<&BStr> { + let source = self.source()?; + let suffix = source.strip_prefix(b"refs/")?; + let slash_pos = suffix.find_byte(b'/')?; + let prefix = source[..="refs/".len() + slash_pos].as_bstr(); + (!prefix.contains(&b'*')).then(|| prefix) + } + /// Transform the state of the refspec into an instruction making clear what to do with it. pub fn instruction(&self) -> Instruction<'_> { match self.op { diff --git a/git-refspec/tests/refspec.rs b/git-refspec/tests/refspec.rs index 5768dc5c1f9..db8e38b6006 100644 --- a/git-refspec/tests/refspec.rs +++ b/git-refspec/tests/refspec.rs @@ -6,4 +6,5 @@ mod impls; mod match_group; mod matching; mod parse; +mod spec; mod write; diff --git a/git-refspec/tests/spec/mod.rs b/git-refspec/tests/spec/mod.rs new file mode 100644 index 00000000000..ca8b9e26575 --- /dev/null +++ b/git-refspec/tests/spec/mod.rs @@ -0,0 +1,41 @@ +mod prefix { + use git_refspec::parse::Operation; + use git_refspec::RefSpec; + + #[test] + fn partial_refs_have_no_prefix() { + assert_eq!(parse("main").to_ref().prefix(), None); + } + + #[test] + fn short_absolute_refs_have_no_prefix() { + assert_eq!(parse("refs/short").to_ref().prefix(), None); + } + + #[test] + fn full_names_have_a_prefix() { + assert_eq!(parse("refs/heads/main").to_ref().prefix().unwrap(), "refs/heads/"); + assert_eq!(parse("refs/foo/bar").to_ref().prefix().unwrap(), "refs/foo/"); + assert_eq!( + parse("refs/heads/*:refs/remotes/origin/*").to_ref().prefix().unwrap(), + "refs/heads/" + ); + } + + #[test] + fn strange_glob_patterns_have_no_prefix() { + assert_eq!(parse("refs/*/main:refs/*/main").to_ref().prefix(), None); + } + + #[test] + fn object_names_have_no_prefix() { + assert_eq!( + parse("e69de29bb2d1d6434b8b29ae775ad8c2e48c5391").to_ref().prefix(), + None + ); + } + + fn parse(spec: &str) -> RefSpec { + git_refspec::parse(spec.into(), Operation::Fetch).unwrap().to_owned() + } +} From 45394d580506722136906426078b333c7ace92eb Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 21 Sep 2022 21:53:25 +0800 Subject: [PATCH 003/113] improve docs a little (#450) --- git-protocol/src/fetch/refs/function.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git-protocol/src/fetch/refs/function.rs b/git-protocol/src/fetch/refs/function.rs index 39f2d039294..0c2df0fdb71 100644 --- a/git-protocol/src/fetch/refs/function.rs +++ b/git-protocol/src/fetch/refs/function.rs @@ -10,7 +10,7 @@ use super::Error; use crate::fetch::{indicate_end_of_interaction, refs::from_v2_refs, Command, LsRefsAction, Ref}; /// Invoke an ls-refs command on `transport` (assuming `protocol_version` 2 or panic), which requires a prior handshake that yielded -/// server `capabilities`. `prepare_ls_refs(arguments, features)` can be used to alter the _ls-refs_. `progress` is used to provide feedback. +/// server `capabilities`. `prepare_ls_refs(capabilities, arguments, features)` can be used to alter the _ls-refs_. `progress` is used to provide feedback. #[maybe_async] pub async fn refs( mut transport: impl Transport, From 3a38d1bc4910aab98c9c0b4a309be4a449db92fc Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 21 Sep 2022 22:01:13 +0800 Subject: [PATCH 004/113] change: turn `prepare_ls_refs` in `fetch::refs()` into `FnOnce`. (#450) --- git-protocol/src/fetch/refs/function.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git-protocol/src/fetch/refs/function.rs b/git-protocol/src/fetch/refs/function.rs index 0c2df0fdb71..e4d2ad7df95 100644 --- a/git-protocol/src/fetch/refs/function.rs +++ b/git-protocol/src/fetch/refs/function.rs @@ -16,7 +16,7 @@ pub async fn refs( mut transport: impl Transport, protocol_version: Protocol, capabilities: &Capabilities, - mut prepare_ls_refs: impl FnMut( + prepare_ls_refs: impl FnOnce( &Capabilities, &mut Vec, &mut Vec<(&str, Option<&str>)>, From 278ff7a6ee084ea864193a5ca25b6cd0f18e19a0 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 21 Sep 2022 22:09:29 +0800 Subject: [PATCH 005/113] fix: `RefSpecRef` instruction uses the correct lifetime. (#450) --- git-refspec/src/spec.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/git-refspec/src/spec.rs b/git-refspec/src/spec.rs index 84657c90d13..ba5b7932698 100644 --- a/git-refspec/src/spec.rs +++ b/git-refspec/src/spec.rs @@ -84,7 +84,7 @@ mod impls { } /// Access -impl RefSpecRef<'_> { +impl<'a> RefSpecRef<'a> { /// Return the left-hand side of the spec, typically the source. /// It takes many different forms so don't rely on this being a ref name. /// @@ -129,7 +129,7 @@ impl RefSpecRef<'_> { } /// Transform the state of the refspec into an instruction making clear what to do with it. - pub fn instruction(&self) -> Instruction<'_> { + pub fn instruction(&self) -> Instruction<'a> { match self.op { Operation::Fetch => match (self.mode, self.src, self.dst) { (Mode::Normal | Mode::Force, Some(src), None) => Instruction::Fetch(Fetch::Only { src }), From 25f06400c49ddd1688fd76f9285542b121b223b4 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 21 Sep 2022 22:09:50 +0800 Subject: [PATCH 006/113] feat: `Remote::rem_map()` now specifies ref-prefixes to the remote. (#450) This can greatly reduce the amount of refs sent. --- .../src/remote/connection/ref_map.rs | 18 +++++++++++++++++- git-repository/tests/remote/mod.rs | 2 +- .../tests/remote/{list_refs.rs => ref_map.rs} | 0 3 files changed, 18 insertions(+), 2 deletions(-) rename git-repository/tests/remote/{list_refs.rs => ref_map.rs} (100%) diff --git a/git-repository/src/remote/connection/ref_map.rs b/git-repository/src/remote/connection/ref_map.rs index 307976f4991..1a9b081bdae 100644 --- a/git-repository/src/remote/connection/ref_map.rs +++ b/git-repository/src/remote/connection/ref_map.rs @@ -1,5 +1,7 @@ +use crate::bstr::{BString, ByteVec}; use git_features::progress::Progress; use git_protocol::transport::client::Transport; +use std::collections::HashSet; use crate::remote::{connection::HandshakeWithRefs, fetch, Connection, Direction}; @@ -99,11 +101,25 @@ where let refs = match outcome.refs.take() { Some(refs) => refs, None => { + let specs = &self.remote.fetch_specs; git_protocol::fetch::refs( &mut self.transport, outcome.server_protocol_version, &outcome.capabilities, - |_a, _b, _c| Ok(git_protocol::fetch::delegate::LsRefsAction::Continue), + |_capabilities, arguments, _features| { + let mut seen = HashSet::new(); + for spec in specs { + let spec = spec.to_ref(); + if seen.insert(spec.instruction()) { + if let Some(prefix) = spec.prefix() { + let mut arg: BString = "ref-prefix ".into(); + arg.push_str(prefix); + arguments.push(arg) + } + } + } + Ok(git_protocol::fetch::delegate::LsRefsAction::Continue) + }, &mut self.progress, ) .await? diff --git a/git-repository/tests/remote/mod.rs b/git-repository/tests/remote/mod.rs index c418f7ed926..9a73832179d 100644 --- a/git-repository/tests/remote/mod.rs +++ b/git-repository/tests/remote/mod.rs @@ -17,4 +17,4 @@ pub(crate) fn cow_str(s: &str) -> Cow { } mod connect; -mod list_refs; +mod ref_map; diff --git a/git-repository/tests/remote/list_refs.rs b/git-repository/tests/remote/ref_map.rs similarity index 100% rename from git-repository/tests/remote/list_refs.rs rename to git-repository/tests/remote/ref_map.rs From 38373bc61c938d58a9d6ed1feae86ccf36fde67d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 21 Sep 2022 22:26:24 +0800 Subject: [PATCH 007/113] Allow to turn remote-filtering off. (#450) That way, we can still see all remote refs if desired. --- .../src/remote/connection/ref_map.rs | 49 ++++++++++++++----- git-repository/tests/remote/ref_map.rs | 2 +- gitoxide-core/src/repository/remote.rs | 4 +- 3 files changed, 40 insertions(+), 15 deletions(-) diff --git a/git-repository/src/remote/connection/ref_map.rs b/git-repository/src/remote/connection/ref_map.rs index 1a9b081bdae..68611decac6 100644 --- a/git-repository/src/remote/connection/ref_map.rs +++ b/git-repository/src/remote/connection/ref_map.rs @@ -21,6 +21,22 @@ pub enum Error { MappingValidation(#[from] git_refspec::match_group::validate::Error), } +/// For use in [`Connection::ref_map()`]. +#[derive(Debug, Copy, Clone)] +pub struct Options { + /// Use a two-component prefix derived from the ref-spec's source, like `refs/heads/` to let the server pre-filter refs + /// with great potential for savings in traffic and local CPU time. + pub prefix_from_spec_as_filter_on_remote: bool, +} + +impl Default for Options { + fn default() -> Self { + Options { + prefix_from_spec_as_filter_on_remote: true, + } + } +} + impl<'remote, 'repo, T, P> Connection<'remote, 'repo, T, P> where T: Transport, @@ -34,15 +50,20 @@ where /// /// Note that this doesn't fetch the objects mentioned in the tips nor does it make any change to underlying repository. #[git_protocol::maybe_async::maybe_async] - pub async fn ref_map(mut self) -> Result, Error> { - let res = self.ref_map_inner().await; + pub async fn ref_map(mut self, options: Options) -> Result, Error> { + let res = self.ref_map_inner(options).await; git_protocol::fetch::indicate_end_of_interaction(&mut self.transport).await?; res } #[git_protocol::maybe_async::maybe_async] - async fn ref_map_inner(&mut self) -> Result, Error> { - let remote = self.fetch_refs().await?; + async fn ref_map_inner( + &mut self, + Options { + prefix_from_spec_as_filter_on_remote, + }: Options, + ) -> Result, Error> { + let remote = self.fetch_refs(prefix_from_spec_as_filter_on_remote).await?; let group = git_refspec::MatchGroup::from_fetch_specs(self.remote.fetch_specs.iter().map(|s| s.to_ref())); let (res, fixes) = group .match_remotes(remote.refs.iter().map(|r| { @@ -79,7 +100,7 @@ where }) } #[git_protocol::maybe_async::maybe_async] - async fn fetch_refs(&mut self) -> Result { + async fn fetch_refs(&mut self, filter_by_prefix: bool) -> Result { let mut credentials_storage; let authenticate = match self.credentials.as_mut() { Some(f) => f, @@ -107,14 +128,16 @@ where outcome.server_protocol_version, &outcome.capabilities, |_capabilities, arguments, _features| { - let mut seen = HashSet::new(); - for spec in specs { - let spec = spec.to_ref(); - if seen.insert(spec.instruction()) { - if let Some(prefix) = spec.prefix() { - let mut arg: BString = "ref-prefix ".into(); - arg.push_str(prefix); - arguments.push(arg) + if filter_by_prefix { + let mut seen = HashSet::new(); + for spec in specs { + let spec = spec.to_ref(); + if seen.insert(spec.instruction()) { + if let Some(prefix) = spec.prefix() { + let mut arg: BString = "ref-prefix ".into(); + arg.push_str(prefix); + arguments.push(arg) + } } } } diff --git a/git-repository/tests/remote/ref_map.rs b/git-repository/tests/remote/ref_map.rs index 67f84673678..6670392e573 100644 --- a/git-repository/tests/remote/ref_map.rs +++ b/git-repository/tests/remote/ref_map.rs @@ -25,7 +25,7 @@ mod blocking_io { let remote = repo.find_remote("origin")?; let connection = remote.connect(Fetch, progress::Discard)?; - let map = connection.ref_map()?; + let map = connection.ref_map(Default::default())?; assert_eq!( map.remote_refs.len(), 14, diff --git a/gitoxide-core/src/repository/remote.rs b/gitoxide-core/src/repository/remote.rs index 81e4fbf8599..a51b5cc1641 100644 --- a/gitoxide-core/src/repository/remote.rs +++ b/gitoxide-core/src/repository/remote.rs @@ -73,7 +73,9 @@ mod refs_impl { let map = remote .connect(git::remote::Direction::Fetch, progress) .await? - .ref_map() + .ref_map(git::remote::ref_map::Options { + prefix_from_spec_as_filter_on_remote: !matches!(kind, refs::Kind::Remote), + }) .await?; if handshake_info { From 02245a619df3a88214b2fb23c3049eedd3ece332 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 22 Sep 2022 09:51:23 +0800 Subject: [PATCH 008/113] update fetch feature list with `wait-for-done` (#450) The latter could alter the way protocol interactions work, but we might already do it correctly. --- git-protocol/src/fetch/command.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/git-protocol/src/fetch/command.rs b/git-protocol/src/fetch/command.rs index c86062dd4cc..162134d8eba 100644 --- a/git-protocol/src/fetch/command.rs +++ b/git-protocol/src/fetch/command.rs @@ -50,9 +50,12 @@ mod with_io { "filter ", // filter-spec // ref-in-want feature "want-ref ", // ref path + // sideband-all feature "sideband-all", // packfile-uris feature "packfile-uris ", // protocols + // wait-for-done feature + "wait-for-done", ], } } @@ -79,9 +82,14 @@ mod with_io { "no-done", "filter", ], - git_transport::Protocol::V2 => { - &["shallow", "filter", "ref-in-want", "sideband-all", "packfile-uris"] - } + git_transport::Protocol::V2 => &[ + "shallow", + "filter", + "ref-in-want", + "sideband-all", + "packfile-uris", + "wait-for-done", + ], }, } } @@ -104,6 +112,7 @@ mod with_io { } } + /// Turns on all modern features for V1 and all supported features for V2. pub(crate) fn default_features( &self, version: git_transport::Protocol, From c3e4b2a9e7dd27d53426e30607ef7a158fe67cc3 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 22 Sep 2022 10:11:24 +0800 Subject: [PATCH 009/113] refactor (#450) --- git-protocol/src/fetch/response/mod.rs | 3 ++- git-protocol/src/remote_progress.rs | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/git-protocol/src/fetch/response/mod.rs b/git-protocol/src/fetch/response/mod.rs index 37e0943ecb8..c9afce53470 100644 --- a/git-protocol/src/fetch/response/mod.rs +++ b/git-protocol/src/fetch/response/mod.rs @@ -152,7 +152,8 @@ impl Response { self.has_pack } - /// Return an error if the given `features` don't contain the required ones for the given `version` of the protocol. + /// Return an error if the given `features` don't contain the required ones (the ones this implementation needs) + /// for the given `version` of the protocol. /// /// Even though technically any set of features supported by the server could work, we only implement the ones that /// make it easy to maintain all versions with a single code base that aims to be and remain maintainable. diff --git a/git-protocol/src/remote_progress.rs b/git-protocol/src/remote_progress.rs index 884057bfbd9..75700c24044 100644 --- a/git-protocol/src/remote_progress.rs +++ b/git-protocol/src/remote_progress.rs @@ -54,7 +54,7 @@ impl<'a> RemoteProgress<'a> { progress.fail(progress_name(None, text)); } } else { - match Self::from_bytes(text) { + match RemoteProgress::from_bytes(text) { Some(RemoteProgress { action, percent: _, From 5475cc2e60dd1cde3ecb24ccf873bc06421f09c9 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 22 Sep 2022 10:11:33 +0800 Subject: [PATCH 010/113] pass extra handshake parameters via options in `ref-map` (#450) --- .../src/remote/connection/ref_map.rs | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/git-repository/src/remote/connection/ref_map.rs b/git-repository/src/remote/connection/ref_map.rs index 68611decac6..645fb966ffe 100644 --- a/git-repository/src/remote/connection/ref_map.rs +++ b/git-repository/src/remote/connection/ref_map.rs @@ -22,17 +22,22 @@ pub enum Error { } /// For use in [`Connection::ref_map()`]. -#[derive(Debug, Copy, Clone)] +#[derive(Debug, Clone)] pub struct Options { /// Use a two-component prefix derived from the ref-spec's source, like `refs/heads/` to let the server pre-filter refs /// with great potential for savings in traffic and local CPU time. pub prefix_from_spec_as_filter_on_remote: bool, + /// Parameters in the form of `(name, optional value)` to add to the handshake. + /// + /// This is useful in case of custom servers. + pub handshake_parameters: Vec<(String, Option)>, } impl Default for Options { fn default() -> Self { Options { prefix_from_spec_as_filter_on_remote: true, + handshake_parameters: Default::default(), } } } @@ -61,9 +66,12 @@ where &mut self, Options { prefix_from_spec_as_filter_on_remote, + handshake_parameters, }: Options, ) -> Result, Error> { - let remote = self.fetch_refs(prefix_from_spec_as_filter_on_remote).await?; + let remote = self + .fetch_refs(prefix_from_spec_as_filter_on_remote, handshake_parameters) + .await?; let group = git_refspec::MatchGroup::from_fetch_specs(self.remote.fetch_specs.iter().map(|s| s.to_ref())); let (res, fixes) = group .match_remotes(remote.refs.iter().map(|r| { @@ -100,7 +108,11 @@ where }) } #[git_protocol::maybe_async::maybe_async] - async fn fetch_refs(&mut self, filter_by_prefix: bool) -> Result { + async fn fetch_refs( + &mut self, + filter_by_prefix: bool, + extra_parameters: Vec<(String, Option)>, + ) -> Result { let mut credentials_storage; let authenticate = match self.credentials.as_mut() { Some(f) => f, @@ -118,7 +130,8 @@ where } }; let mut outcome = - git_protocol::fetch::handshake(&mut self.transport, authenticate, Vec::new(), &mut self.progress).await?; + git_protocol::fetch::handshake(&mut self.transport, authenticate, extra_parameters, &mut self.progress) + .await?; let refs = match outcome.refs.take() { Some(refs) => refs, None => { From 0dc7206ee0dff760d632362de3b41d9e4dc22598 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 22 Sep 2022 12:48:18 +0800 Subject: [PATCH 011/113] first sktech of fetch module (#450) --- git-repository/src/remote/connection/fetch.rs | 17 +++++++++++++++++ git-repository/src/remote/connection/mod.rs | 4 ++++ git-repository/src/remote/connection/ref_map.rs | 5 +++++ git-repository/src/remote/mod.rs | 3 +++ 4 files changed, 29 insertions(+) create mode 100644 git-repository/src/remote/connection/fetch.rs diff --git a/git-repository/src/remote/connection/fetch.rs b/git-repository/src/remote/connection/fetch.rs new file mode 100644 index 00000000000..d723c917a67 --- /dev/null +++ b/git-repository/src/remote/connection/fetch.rs @@ -0,0 +1,17 @@ +use crate::remote::Connection; +use crate::Progress; +use git_protocol::transport::client::Transport; + +#[allow(missing_docs)] +pub struct Options {} + +impl<'remote, 'repo, T, P> Connection<'remote, 'repo, T, P> +where + T: Transport, + P: Progress, +{ + #[allow(missing_docs)] + pub fn prepare_fetch(self) -> ! { + todo!() + } +} diff --git a/git-repository/src/remote/connection/mod.rs b/git-repository/src/remote/connection/mod.rs index 0ac97e17902..6559f41abf5 100644 --- a/git-repository/src/remote/connection/mod.rs +++ b/git-repository/src/remote/connection/mod.rs @@ -67,3 +67,7 @@ mod access { /// pub mod ref_map; + +/// +#[cfg(feature = "blocking-network-client")] +pub mod fetch; diff --git a/git-repository/src/remote/connection/ref_map.rs b/git-repository/src/remote/connection/ref_map.rs index 645fb966ffe..92dfe5bb7ca 100644 --- a/git-repository/src/remote/connection/ref_map.rs +++ b/git-repository/src/remote/connection/ref_map.rs @@ -54,6 +54,11 @@ where /// with the local tracking branch of these tips (if available). /// /// Note that this doesn't fetch the objects mentioned in the tips nor does it make any change to underlying repository. + /// + /// # Consumption + /// + /// Due to management of the transport, it's cleanest to only use it for a single interaction. Thus it's consumed along with + /// the connection. #[git_protocol::maybe_async::maybe_async] pub async fn ref_map(mut self, options: Options) -> Result, Error> { let res = self.ref_map_inner(options).await; diff --git a/git-repository/src/remote/mod.rs b/git-repository/src/remote/mod.rs index e1bed940331..327c359fb23 100644 --- a/git-repository/src/remote/mod.rs +++ b/git-repository/src/remote/mod.rs @@ -64,6 +64,9 @@ pub mod fetch { /// The index into the fetch ref-specs used to produce the mapping, allowing it to be recovered. pub spec_index: usize, } + + #[cfg(feature = "blocking-network-client")] + pub use super::connection::fetch::Options; } /// From 5bef0a00e8d01110c054a517f6d9696f981a7efc Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 22 Sep 2022 14:11:01 +0800 Subject: [PATCH 012/113] =?UTF-8?q?FAIL:=20try=20to=20make=20the=20transpo?= =?UTF-8?q?rt=20configurable=20after=20being=20boxed,=20but=E2=80=A6=20(#4?= =?UTF-8?q?50)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …that would force it to be 'static, which is something we excplicitly cannot have. We need references to be contained within, if I remember correctly. --- git-repository/src/remote/connection/mod.rs | 26 +++++++++++++++++++ .../src/remote/connection/ref_map.rs | 6 ++--- git-transport/src/client/blocking_io/file.rs | 5 ++++ .../src/client/blocking_io/http/mod.rs | 5 ++++ git-transport/src/client/git/blocking_io.rs | 5 ++++ git-transport/src/client/traits.rs | 14 ++++++++++ 6 files changed, 58 insertions(+), 3 deletions(-) diff --git a/git-repository/src/remote/connection/mod.rs b/git-repository/src/remote/connection/mod.rs index 6559f41abf5..42939f612dd 100644 --- a/git-repository/src/remote/connection/mod.rs +++ b/git-repository/src/remote/connection/mod.rs @@ -24,6 +24,7 @@ mod access { remote::{connection::CredentialsFn, Connection}, Remote, }; + use std::any::Any; /// Builder impl<'a, 'repo, T, P> Connection<'a, 'repo, T, P> { @@ -62,6 +63,31 @@ mod access { pub fn remote(&self) -> &Remote<'repo> { self.remote } + + /// Return the connection's transport. + pub fn transport(&self) -> &T { + &self.transport + } + } + + /// Access to the transport if it can be downcast to a particular type. + impl<'a, 'repo, T, P> Connection<'a, 'repo, T, P> + where + T: crate::protocol::transport::client::Transport + 'static, + { + /// Try to cast our transport `T` into `U`, and pass it to `f` to allow any kind of configuration. + /// + /// Note that if the case fails and `f` is not called at all, `false` is returned. + pub fn configure_transport( + &mut self, + f: impl FnOnce(&mut U) -> Result<(), E>, + ) -> Result { + let transport = (&mut self.transport) as &mut dyn Any; + match transport.downcast_mut::() { + Some(transport) => f(transport).map(|_| true), + None => Ok(false), + } + } } } diff --git a/git-repository/src/remote/connection/ref_map.rs b/git-repository/src/remote/connection/ref_map.rs index 92dfe5bb7ca..965db7a8346 100644 --- a/git-repository/src/remote/connection/ref_map.rs +++ b/git-repository/src/remote/connection/ref_map.rs @@ -134,15 +134,15 @@ where &mut credentials_storage } }; + let transport = &mut self.transport; let mut outcome = - git_protocol::fetch::handshake(&mut self.transport, authenticate, extra_parameters, &mut self.progress) - .await?; + git_protocol::fetch::handshake(&mut *transport, authenticate, extra_parameters, &mut self.progress).await?; let refs = match outcome.refs.take() { Some(refs) => refs, None => { let specs = &self.remote.fetch_specs; git_protocol::fetch::refs( - &mut self.transport, + transport, outcome.server_protocol_version, &outcome.capabilities, |_capabilities, arguments, _features| { diff --git a/git-transport/src/client/blocking_io/file.rs b/git-transport/src/client/blocking_io/file.rs index 0e23075acd2..e5cccb83cf3 100644 --- a/git-transport/src/client/blocking_io/file.rs +++ b/git-transport/src/client/blocking_io/file.rs @@ -1,3 +1,4 @@ +use std::any::Any; use std::process::{self, Command, Stdio}; use bstr::{BString, ByteSlice}; @@ -83,6 +84,10 @@ impl SpawnProcessOnDemand { } impl client::TransportWithoutIO for SpawnProcessOnDemand { + fn as_any_mut(&mut self) -> &mut dyn Any { + self + } + fn request( &mut self, write_mode: WriteMode, diff --git a/git-transport/src/client/blocking_io/http/mod.rs b/git-transport/src/client/blocking_io/http/mod.rs index 8c6b5269fdf..947e2ed4a22 100644 --- a/git-transport/src/client/blocking_io/http/mod.rs +++ b/git-transport/src/client/blocking_io/http/mod.rs @@ -1,3 +1,4 @@ +use std::any::Any; use std::{ borrow::Cow, convert::Infallible, @@ -99,6 +100,10 @@ fn append_url(base: &str, suffix: &str) -> String { } impl client::TransportWithoutIO for Transport { + fn as_any_mut(&mut self) -> &mut dyn Any { + self + } + fn set_identity(&mut self, identity: git_sec::identity::Account) -> Result<(), client::Error> { self.identity = Some(identity); Ok(()) diff --git a/git-transport/src/client/git/blocking_io.rs b/git-transport/src/client/git/blocking_io.rs index 240f00c9903..f8fb6acbaf4 100644 --- a/git-transport/src/client/git/blocking_io.rs +++ b/git-transport/src/client/git/blocking_io.rs @@ -1,3 +1,4 @@ +use std::any::Any; use std::io::Write; use bstr::BString; @@ -13,6 +14,10 @@ where R: std::io::Read, W: std::io::Write, { + fn as_any_mut(&mut self) -> &mut dyn Any { + self + } + fn request( &mut self, write_mode: client::WriteMode, diff --git a/git-transport/src/client/traits.rs b/git-transport/src/client/traits.rs index f57b2fe2d09..7e82babb207 100644 --- a/git-transport/src/client/traits.rs +++ b/git-transport/src/client/traits.rs @@ -1,3 +1,4 @@ +use std::any::Any; use std::ops::{Deref, DerefMut}; #[cfg(any(feature = "blocking-client", feature = "async-client"))] @@ -7,6 +8,11 @@ use crate::{client::Error, Protocol}; /// This trait represents all transport related functions that don't require any input/output to be done which helps /// implementation to share more code across blocking and async programs. pub trait TransportWithoutIO { + /// Cast this type as dyn trait to allow casting it back to a known concrete type. + /// + /// This is useful for configuration after a boxed transport implementation has been crated. + fn as_any_mut(&mut self) -> &mut dyn std::any::Any; + /// If the handshake or subsequent reads failed with [std::io::ErrorKind::PermissionDenied], use this method to /// inform the transport layer about the identity to use for subsequent calls. /// If authentication continues to fail even with an identity set, consider communicating this to the provider @@ -50,6 +56,10 @@ pub trait TransportWithoutIO { // Would be nice if the box implementation could auto-forward to all implemented traits. impl TransportWithoutIO for Box { + fn as_any_mut(&mut self) -> &mut dyn Any { + self + } + fn set_identity(&mut self, identity: git_sec::identity::Account) -> Result<(), Error> { self.deref_mut().set_identity(identity) } @@ -73,6 +83,10 @@ impl TransportWithoutIO for Box { } impl TransportWithoutIO for &mut T { + fn as_any_mut(&mut self) -> &mut dyn Any { + self + } + fn set_identity(&mut self, identity: git_sec::identity::Account) -> Result<(), Error> { self.deref_mut().set_identity(identity) } From fbb96e4d55e322243bf5500605f72e93b103e308 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 22 Sep 2022 14:12:04 +0800 Subject: [PATCH 013/113] =?UTF-8?q?Revert=20"FAIL:=20try=20to=20make=20the?= =?UTF-8?q?=20transport=20configurable=20after=20being=20boxed,=20but?= =?UTF-8?q?=E2=80=A6=20(#450)"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 5bef0a00e8d01110c054a517f6d9696f981a7efc. --- git-repository/src/remote/connection/mod.rs | 26 ------------------- .../src/remote/connection/ref_map.rs | 6 ++--- git-transport/src/client/blocking_io/file.rs | 5 ---- .../src/client/blocking_io/http/mod.rs | 5 ---- git-transport/src/client/git/blocking_io.rs | 5 ---- git-transport/src/client/traits.rs | 14 ---------- 6 files changed, 3 insertions(+), 58 deletions(-) diff --git a/git-repository/src/remote/connection/mod.rs b/git-repository/src/remote/connection/mod.rs index 42939f612dd..6559f41abf5 100644 --- a/git-repository/src/remote/connection/mod.rs +++ b/git-repository/src/remote/connection/mod.rs @@ -24,7 +24,6 @@ mod access { remote::{connection::CredentialsFn, Connection}, Remote, }; - use std::any::Any; /// Builder impl<'a, 'repo, T, P> Connection<'a, 'repo, T, P> { @@ -63,31 +62,6 @@ mod access { pub fn remote(&self) -> &Remote<'repo> { self.remote } - - /// Return the connection's transport. - pub fn transport(&self) -> &T { - &self.transport - } - } - - /// Access to the transport if it can be downcast to a particular type. - impl<'a, 'repo, T, P> Connection<'a, 'repo, T, P> - where - T: crate::protocol::transport::client::Transport + 'static, - { - /// Try to cast our transport `T` into `U`, and pass it to `f` to allow any kind of configuration. - /// - /// Note that if the case fails and `f` is not called at all, `false` is returned. - pub fn configure_transport( - &mut self, - f: impl FnOnce(&mut U) -> Result<(), E>, - ) -> Result { - let transport = (&mut self.transport) as &mut dyn Any; - match transport.downcast_mut::() { - Some(transport) => f(transport).map(|_| true), - None => Ok(false), - } - } } } diff --git a/git-repository/src/remote/connection/ref_map.rs b/git-repository/src/remote/connection/ref_map.rs index 965db7a8346..92dfe5bb7ca 100644 --- a/git-repository/src/remote/connection/ref_map.rs +++ b/git-repository/src/remote/connection/ref_map.rs @@ -134,15 +134,15 @@ where &mut credentials_storage } }; - let transport = &mut self.transport; let mut outcome = - git_protocol::fetch::handshake(&mut *transport, authenticate, extra_parameters, &mut self.progress).await?; + git_protocol::fetch::handshake(&mut self.transport, authenticate, extra_parameters, &mut self.progress) + .await?; let refs = match outcome.refs.take() { Some(refs) => refs, None => { let specs = &self.remote.fetch_specs; git_protocol::fetch::refs( - transport, + &mut self.transport, outcome.server_protocol_version, &outcome.capabilities, |_capabilities, arguments, _features| { diff --git a/git-transport/src/client/blocking_io/file.rs b/git-transport/src/client/blocking_io/file.rs index e5cccb83cf3..0e23075acd2 100644 --- a/git-transport/src/client/blocking_io/file.rs +++ b/git-transport/src/client/blocking_io/file.rs @@ -1,4 +1,3 @@ -use std::any::Any; use std::process::{self, Command, Stdio}; use bstr::{BString, ByteSlice}; @@ -84,10 +83,6 @@ impl SpawnProcessOnDemand { } impl client::TransportWithoutIO for SpawnProcessOnDemand { - fn as_any_mut(&mut self) -> &mut dyn Any { - self - } - fn request( &mut self, write_mode: WriteMode, diff --git a/git-transport/src/client/blocking_io/http/mod.rs b/git-transport/src/client/blocking_io/http/mod.rs index 947e2ed4a22..8c6b5269fdf 100644 --- a/git-transport/src/client/blocking_io/http/mod.rs +++ b/git-transport/src/client/blocking_io/http/mod.rs @@ -1,4 +1,3 @@ -use std::any::Any; use std::{ borrow::Cow, convert::Infallible, @@ -100,10 +99,6 @@ fn append_url(base: &str, suffix: &str) -> String { } impl client::TransportWithoutIO for Transport { - fn as_any_mut(&mut self) -> &mut dyn Any { - self - } - fn set_identity(&mut self, identity: git_sec::identity::Account) -> Result<(), client::Error> { self.identity = Some(identity); Ok(()) diff --git a/git-transport/src/client/git/blocking_io.rs b/git-transport/src/client/git/blocking_io.rs index f8fb6acbaf4..240f00c9903 100644 --- a/git-transport/src/client/git/blocking_io.rs +++ b/git-transport/src/client/git/blocking_io.rs @@ -1,4 +1,3 @@ -use std::any::Any; use std::io::Write; use bstr::BString; @@ -14,10 +13,6 @@ where R: std::io::Read, W: std::io::Write, { - fn as_any_mut(&mut self) -> &mut dyn Any { - self - } - fn request( &mut self, write_mode: client::WriteMode, diff --git a/git-transport/src/client/traits.rs b/git-transport/src/client/traits.rs index 7e82babb207..f57b2fe2d09 100644 --- a/git-transport/src/client/traits.rs +++ b/git-transport/src/client/traits.rs @@ -1,4 +1,3 @@ -use std::any::Any; use std::ops::{Deref, DerefMut}; #[cfg(any(feature = "blocking-client", feature = "async-client"))] @@ -8,11 +7,6 @@ use crate::{client::Error, Protocol}; /// This trait represents all transport related functions that don't require any input/output to be done which helps /// implementation to share more code across blocking and async programs. pub trait TransportWithoutIO { - /// Cast this type as dyn trait to allow casting it back to a known concrete type. - /// - /// This is useful for configuration after a boxed transport implementation has been crated. - fn as_any_mut(&mut self) -> &mut dyn std::any::Any; - /// If the handshake or subsequent reads failed with [std::io::ErrorKind::PermissionDenied], use this method to /// inform the transport layer about the identity to use for subsequent calls. /// If authentication continues to fail even with an identity set, consider communicating this to the provider @@ -56,10 +50,6 @@ pub trait TransportWithoutIO { // Would be nice if the box implementation could auto-forward to all implemented traits. impl TransportWithoutIO for Box { - fn as_any_mut(&mut self) -> &mut dyn Any { - self - } - fn set_identity(&mut self, identity: git_sec::identity::Account) -> Result<(), Error> { self.deref_mut().set_identity(identity) } @@ -83,10 +73,6 @@ impl TransportWithoutIO for Box { } impl TransportWithoutIO for &mut T { - fn as_any_mut(&mut self) -> &mut dyn Any { - self - } - fn set_identity(&mut self, identity: git_sec::identity::Account) -> Result<(), Error> { self.deref_mut().set_identity(identity) } From 1cf66c4dbc7a0404701efe4335363c2636ce32f8 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 22 Sep 2022 14:42:48 +0800 Subject: [PATCH 014/113] change!: `client::http::connect()` returns `Transport` directly. (#450) It' can't fail, so no need to return `Result<_, Infallible>`. --- git-transport/src/client/blocking_io/connect.rs | 8 ++++---- git-transport/src/client/blocking_io/http/mod.rs | 5 ++--- git-transport/tests/client/blocking_io/http/mock.rs | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/git-transport/src/client/blocking_io/connect.rs b/git-transport/src/client/blocking_io/connect.rs index dc4704da7a4..d42cdb67326 100644 --- a/git-transport/src/client/blocking_io/connect.rs +++ b/git-transport/src/client/blocking_io/connect.rs @@ -66,10 +66,10 @@ pub(crate) mod function { #[cfg(not(feature = "http-client-curl"))] git_url::Scheme::Https | git_url::Scheme::Http => return Err(Error::CompiledWithoutHttp(url.scheme)), #[cfg(feature = "http-client-curl")] - git_url::Scheme::Https | git_url::Scheme::Http => Box::new( - crate::client::http::connect(&url.to_bstring().to_string(), desired_version) - .map_err(|e| Box::new(e) as Box)?, - ), + git_url::Scheme::Https | git_url::Scheme::Http => Box::new(crate::client::http::connect( + &url.to_bstring().to_string(), + desired_version, + )), }) } } diff --git a/git-transport/src/client/blocking_io/http/mod.rs b/git-transport/src/client/blocking_io/http/mod.rs index 8c6b5269fdf..e97e6d5d319 100644 --- a/git-transport/src/client/blocking_io/http/mod.rs +++ b/git-transport/src/client/blocking_io/http/mod.rs @@ -1,6 +1,5 @@ use std::{ borrow::Cow, - convert::Infallible, io::{BufRead, Read}, }; @@ -286,6 +285,6 @@ impl ExtendedBufRead for HeadersThenBody Result, Infallible> { - Ok(Transport::new(url, desired_version)) +pub fn connect(url: &str, desired_version: Protocol) -> Transport { + Transport::new(url, desired_version) } diff --git a/git-transport/tests/client/blocking_io/http/mock.rs b/git-transport/tests/client/blocking_io/http/mock.rs index 49c74ee5e1a..f11f1884bb2 100644 --- a/git-transport/tests/client/blocking_io/http/mock.rs +++ b/git-transport/tests/client/blocking_io/http/mock.rs @@ -104,7 +104,7 @@ pub fn serve_and_connect( &server.addr.port(), path ); - let client = git_transport::client::http::connect(&url, version)?; + let client = git_transport::client::http::connect(&url, version); assert_eq!(url, client.to_url()); Ok((server, client)) } From 211e65d185470ade84a3cc73e1898599b7f15f7c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 22 Sep 2022 14:44:01 +0800 Subject: [PATCH 015/113] Make it easier to connect to http if well-known to allow additional configuration. (#450) --- git-repository/src/remote/connect.rs | 40 +++++++++++++++++++-- git-repository/src/remote/connection/mod.rs | 5 +++ 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/git-repository/src/remote/connect.rs b/git-repository/src/remote/connect.rs index 38aa61cb0b7..1a0a7b48372 100644 --- a/git-repository/src/remote/connect.rs +++ b/git-repository/src/remote/connect.rs @@ -27,6 +27,9 @@ mod error { #[cfg(any(feature = "blocking-network-client", feature = "async-network-client-async-std"))] pub use error::Error; +#[cfg(all(feature = "blocking-http-transport", feature = "blocking-network-client"))] +use git_protocol::transport::client::http; + /// Establishing connections to remote hosts impl<'repo> Remote<'repo> { /// Create a new connection using `transport` to communicate, with `progress` to indicate changes. @@ -60,7 +63,38 @@ impl<'repo> Remote<'repo> { where P: Progress, { - use git_protocol::transport::Protocol; + let (url, version) = self.sanitized_url_and_version(direction)?; + let transport = git_protocol::transport::connect(url, version).await?; + Ok(self.to_connection_with_transport(transport, progress)) + } + + /// Connect to the http(s) url suitable for `direction` and return a handle through which operations can be performed. + /// + /// Note that the `protocol.version` configuration key affects the transport protocol used to connect, + /// with `2` being the default, and that the 'dumb'-http protocol isn't supported. + /// + /// Using this method has the advantage of + #[cfg(all(feature = "blocking-http-transport", feature = "blocking-network-client"))] + pub async fn connect_http

( + &self, + direction: crate::remote::Direction, + progress: P, + ) -> Result, P>, Error> + where + P: Progress, + { + let (url, version) = self.sanitized_url_and_version(direction)?; + let transport = http::connect(&url.to_bstring().to_string(), version); + Ok(self.to_connection_with_transport(transport, progress)) + } + + /// Produce the sanitized URL and protocol version to use as obtained by querying the repository configuration. + /// + /// This can be useful when using custom transports to allow additional configuration. + pub fn sanitized_url_and_version( + &self, + direction: crate::remote::Direction, + ) -> Result<(git_url::Url, git_protocol::transport::Protocol), Error> { fn sanitize(mut url: git_url::Url) -> Result { if url.scheme == git_url::Scheme::File { let mut dir = git_path::from_bstr(url.path.as_ref()); @@ -75,6 +109,7 @@ impl<'repo> Remote<'repo> { Ok(url) } + use git_protocol::transport::Protocol; let version = self .repo .config @@ -101,7 +136,6 @@ impl<'repo> Remote<'repo> { scheme: url.scheme, }); } - let transport = git_protocol::transport::connect(sanitize(url)?, version).await?; - Ok(self.to_connection_with_transport(transport, progress)) + Ok((sanitize(url)?, version)) } } diff --git a/git-repository/src/remote/connection/mod.rs b/git-repository/src/remote/connection/mod.rs index 6559f41abf5..98bcbdc7aa7 100644 --- a/git-repository/src/remote/connection/mod.rs +++ b/git-repository/src/remote/connection/mod.rs @@ -62,6 +62,11 @@ mod access { pub fn remote(&self) -> &Remote<'repo> { self.remote } + + /// Provide a mutable transport to allow configuring it. + pub fn transport_mut(&mut self) -> &mut T { + &mut self.transport + } } } From 47d5cd67b31d4b18c224859b7d9e145c993a4f2d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 22 Sep 2022 15:04:44 +0800 Subject: [PATCH 016/113] don't fail if we can't indicate the end of interaction to the server (#450) After all, this is somewhat optional and the server stops once we disconnect, and the called function may already have performed that action. --- git-repository/src/remote/connect.rs | 2 +- git-repository/src/remote/connection/ref_map.rs | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/git-repository/src/remote/connect.rs b/git-repository/src/remote/connect.rs index 1a0a7b48372..ffe08f0dc01 100644 --- a/git-repository/src/remote/connect.rs +++ b/git-repository/src/remote/connect.rs @@ -30,7 +30,7 @@ pub use error::Error; #[cfg(all(feature = "blocking-http-transport", feature = "blocking-network-client"))] use git_protocol::transport::client::http; -/// Establishing connections to remote hosts +/// Establishing connections to remote hosts (without performing a git-handshake). impl<'repo> Remote<'repo> { /// Create a new connection using `transport` to communicate, with `progress` to indicate changes. /// diff --git a/git-repository/src/remote/connection/ref_map.rs b/git-repository/src/remote/connection/ref_map.rs index 92dfe5bb7ca..8207e686d30 100644 --- a/git-repository/src/remote/connection/ref_map.rs +++ b/git-repository/src/remote/connection/ref_map.rs @@ -62,7 +62,9 @@ where #[git_protocol::maybe_async::maybe_async] pub async fn ref_map(mut self, options: Options) -> Result, Error> { let res = self.ref_map_inner(options).await; - git_protocol::fetch::indicate_end_of_interaction(&mut self.transport).await?; + git_protocol::fetch::indicate_end_of_interaction(&mut self.transport) + .await + .ok(); res } From 744ed03cf6648e258affab62ef39ffad75c65397 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 22 Sep 2022 15:43:26 +0800 Subject: [PATCH 017/113] feat: add `client::fetch_pack()` allowing to fetch a pack after a handshake. (#450) --- git-protocol/src/fetch/delegate.rs | 9 +-- git-protocol/src/fetch/handshake.rs | 2 + git-protocol/src/fetch/refs/function.rs | 2 + git-protocol/src/fetch_fn.rs | 92 ++++++++++++++++++++++++- git-protocol/src/lib.rs | 2 +- 5 files changed, 101 insertions(+), 6 deletions(-) diff --git a/git-protocol/src/fetch/delegate.rs b/git-protocol/src/fetch/delegate.rs index 1d116beb1d3..95d874b0951 100644 --- a/git-protocol/src/fetch/delegate.rs +++ b/git-protocol/src/fetch/delegate.rs @@ -193,7 +193,7 @@ mod blocking_io { use git_features::progress::Progress; - use crate::fetch::{DelegateBlocking, Ref, Response}; + use crate::fetch::{Ref, Response}; /// The protocol delegate is the bare minimal interface needed to fully control the [`fetch`][crate::fetch()] operation. /// @@ -202,7 +202,7 @@ mod blocking_io { /// Everything is tucked away behind type-safety so 'nothing can go wrong'©. Runtime assertions assure invalid /// features or arguments don't make it to the server in the first place. /// Please note that this trait mostly corresponds to what V2 would look like, even though V1 is supported as well. - pub trait Delegate: DelegateBlocking { + pub trait Delegate { /// Receive a pack provided from the given `input`. /// /// Use `progress` to emit your own progress messages when decoding the pack. @@ -253,7 +253,7 @@ mod async_io { use futures_io::AsyncBufRead; use git_features::progress::Progress; - use crate::fetch::{DelegateBlocking, Ref, Response}; + use crate::fetch::{Ref, Response}; /// The protocol delegate is the bare minimal interface needed to fully control the [`fetch`][crate::fetch()] operation. /// @@ -263,7 +263,7 @@ mod async_io { /// features or arguments don't make it to the server in the first place. /// Please note that this trait mostly corresponds to what V2 would look like, even though V1 is supported as well. #[async_trait(?Send)] - pub trait Delegate: DelegateBlocking { + pub trait Delegate { /// Receive a pack provided from the given `input`, and the caller should consider it to be blocking as /// most operations on the received pack are implemented in a blocking fashion. /// @@ -279,6 +279,7 @@ mod async_io { previous_response: &Response, ) -> io::Result<()>; } + #[async_trait(?Send)] impl Delegate for Box { async fn receive_pack( diff --git a/git-protocol/src/fetch/handshake.rs b/git-protocol/src/fetch/handshake.rs index 78d71095339..308f9cfbbaf 100644 --- a/git-protocol/src/fetch/handshake.rs +++ b/git-protocol/src/fetch/handshake.rs @@ -47,6 +47,8 @@ pub(crate) mod function { /// turns out to be required. `extra_parameters` are the parameters `(name, optional value)` to add to the handshake, /// each time it is performed in case authentication is required. /// `progress` is used to inform about what's currently happening. + /// + /// Note that this function never terminates an existing connection on error as it is assumed to be mutually invalid. #[maybe_async] pub async fn handshake( mut transport: T, diff --git a/git-protocol/src/fetch/refs/function.rs b/git-protocol/src/fetch/refs/function.rs index e4d2ad7df95..2d11831ccab 100644 --- a/git-protocol/src/fetch/refs/function.rs +++ b/git-protocol/src/fetch/refs/function.rs @@ -11,6 +11,8 @@ use crate::fetch::{indicate_end_of_interaction, refs::from_v2_refs, Command, LsR /// Invoke an ls-refs command on `transport` (assuming `protocol_version` 2 or panic), which requires a prior handshake that yielded /// server `capabilities`. `prepare_ls_refs(capabilities, arguments, features)` can be used to alter the _ls-refs_. `progress` is used to provide feedback. +/// +/// Note that this function can be assumed to terminate the connection (as indicated to the server) if `prepare_ls_refs` has an error. #[maybe_async] pub async fn refs( mut transport: impl Transport, diff --git a/git-protocol/src/fetch_fn.rs b/git-protocol/src/fetch_fn.rs index e19d78deca8..44ee175ed98 100644 --- a/git-protocol/src/fetch_fn.rs +++ b/git-protocol/src/fetch_fn.rs @@ -1,7 +1,9 @@ use git_features::progress::Progress; use git_transport::client; +use git_transport::client::Capabilities; use maybe_async::maybe_async; +use crate::fetch::{DelegateBlocking, Ref}; use crate::{ credentials, fetch::{handshake, indicate_end_of_interaction, Action, Arguments, Command, Delegate, Error, Response}, @@ -53,7 +55,7 @@ pub async fn fetch( ) -> Result<(), Error> where F: FnMut(credentials::helper::Action) -> credentials::protocol::Result, - D: Delegate, + D: Delegate + DelegateBlocking, T: client::Transport, { let handshake::Outcome { @@ -141,6 +143,94 @@ where Ok(()) } +/// Fetch the pack after a `handshake` has been performed on `transport` and `refs` have been obtained. +/// +/// `negotiate` maybe called repeatedly, after `prepare_fetch` was called once, to setup the haves and wants to fetch. +/// The `delegate` will receive a reader with pack data when done, and `progress` sees all remote end progress messages. +#[maybe_async] +pub async fn fetch_pack( + handshake::Outcome { + server_protocol_version: protocol_version, + refs: _, + capabilities, + }: &handshake::Outcome, + refs: &[Ref], + mut transport: T, + mut prepare_fetch: impl FnMut( + git_transport::Protocol, + &Capabilities, + &mut Vec<(&str, Option<&str>)>, + &[Ref], + ) -> std::io::Result, + mut negotiate: impl FnMut(&[Ref], &mut Arguments, Option<&Response>) -> std::io::Result, + mut delegate: D, + mut progress: impl Progress, + fetch_mode: FetchConnection, +) -> Result<(), Error> +where + F: FnMut(credentials::helper::Action) -> credentials::protocol::Result, + D: Delegate, + T: client::Transport, +{ + let fetch = Command::Fetch; + let mut fetch_features = fetch.default_features(*protocol_version, capabilities); + match prepare_fetch(*protocol_version, capabilities, &mut fetch_features, refs) { + Ok(Action::Cancel) => { + return if matches!(protocol_version, git_transport::Protocol::V1) + || matches!(fetch_mode, FetchConnection::TerminateOnSuccessfulCompletion) + { + indicate_end_of_interaction(transport).await.map_err(Into::into) + } else { + Ok(()) + }; + } + Ok(Action::Continue) => { + fetch.validate_argument_prefixes_or_panic(*protocol_version, capabilities, &[], &fetch_features); + } + Err(err) => { + indicate_end_of_interaction(transport).await?; + return Err(err.into()); + } + } + + Response::check_required_features(*protocol_version, &fetch_features)?; + let sideband_all = fetch_features.iter().any(|(n, _)| *n == "sideband-all"); + let mut arguments = Arguments::new(*protocol_version, fetch_features); + let mut previous_response = None::; + let mut round = 1; + 'negotiation: loop { + progress.step(); + progress.set_name(format!("negotiate (round {})", round)); + round += 1; + let action = negotiate(refs, &mut arguments, previous_response.as_ref())?; + let mut reader = arguments.send(&mut transport, action == Action::Cancel).await?; + if sideband_all { + setup_remote_progress(&mut progress, &mut reader); + } + let response = Response::from_line_reader(*protocol_version, &mut reader).await?; + previous_response = if response.has_pack() { + progress.step(); + progress.set_name("receiving pack"); + if !sideband_all { + setup_remote_progress(&mut progress, &mut reader); + } + delegate.receive_pack(reader, progress, refs, &response).await?; + break 'negotiation; + } else { + match action { + Action::Cancel => break 'negotiation, + Action::Continue => Some(response), + } + } + } + if matches!(protocol_version, git_transport::Protocol::V2) + && matches!(fetch_mode, FetchConnection::TerminateOnSuccessfulCompletion) + { + indicate_end_of_interaction(transport).await?; + } + Ok(()) +} + fn setup_remote_progress( progress: &mut impl Progress, reader: &mut Box, diff --git a/git-protocol/src/lib.rs b/git-protocol/src/lib.rs index 12c69e87471..279a02458bb 100644 --- a/git-protocol/src/lib.rs +++ b/git-protocol/src/lib.rs @@ -26,7 +26,7 @@ pub mod fetch; #[cfg(any(feature = "blocking-client", feature = "async-client"))] mod fetch_fn; #[cfg(any(feature = "blocking-client", feature = "async-client"))] -pub use fetch_fn::{fetch, FetchConnection}; +pub use fetch_fn::{fetch, fetch_pack, FetchConnection}; mod remote_progress; pub use remote_progress::RemoteProgress; From f87b7eb4dc05b73f33ac554b57d50f3eee7f84c1 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 22 Sep 2022 16:14:01 +0800 Subject: [PATCH 018/113] remove: `client::fetch_pack()` isn't worth it after all. (#450) The overhead required to make it work isn't worth the logic we are enabling. Instead, it's OK to re-implement custom logic on the consumer side based on this one. --- git-protocol/src/fetch/delegate.rs | 9 ++- git-protocol/src/fetch/handshake.rs | 2 - git-protocol/src/fetch/refs/function.rs | 2 - git-protocol/src/fetch_fn.rs | 92 +------------------------ git-protocol/src/lib.rs | 2 +- 5 files changed, 6 insertions(+), 101 deletions(-) diff --git a/git-protocol/src/fetch/delegate.rs b/git-protocol/src/fetch/delegate.rs index 95d874b0951..1d116beb1d3 100644 --- a/git-protocol/src/fetch/delegate.rs +++ b/git-protocol/src/fetch/delegate.rs @@ -193,7 +193,7 @@ mod blocking_io { use git_features::progress::Progress; - use crate::fetch::{Ref, Response}; + use crate::fetch::{DelegateBlocking, Ref, Response}; /// The protocol delegate is the bare minimal interface needed to fully control the [`fetch`][crate::fetch()] operation. /// @@ -202,7 +202,7 @@ mod blocking_io { /// Everything is tucked away behind type-safety so 'nothing can go wrong'©. Runtime assertions assure invalid /// features or arguments don't make it to the server in the first place. /// Please note that this trait mostly corresponds to what V2 would look like, even though V1 is supported as well. - pub trait Delegate { + pub trait Delegate: DelegateBlocking { /// Receive a pack provided from the given `input`. /// /// Use `progress` to emit your own progress messages when decoding the pack. @@ -253,7 +253,7 @@ mod async_io { use futures_io::AsyncBufRead; use git_features::progress::Progress; - use crate::fetch::{Ref, Response}; + use crate::fetch::{DelegateBlocking, Ref, Response}; /// The protocol delegate is the bare minimal interface needed to fully control the [`fetch`][crate::fetch()] operation. /// @@ -263,7 +263,7 @@ mod async_io { /// features or arguments don't make it to the server in the first place. /// Please note that this trait mostly corresponds to what V2 would look like, even though V1 is supported as well. #[async_trait(?Send)] - pub trait Delegate { + pub trait Delegate: DelegateBlocking { /// Receive a pack provided from the given `input`, and the caller should consider it to be blocking as /// most operations on the received pack are implemented in a blocking fashion. /// @@ -279,7 +279,6 @@ mod async_io { previous_response: &Response, ) -> io::Result<()>; } - #[async_trait(?Send)] impl Delegate for Box { async fn receive_pack( diff --git a/git-protocol/src/fetch/handshake.rs b/git-protocol/src/fetch/handshake.rs index 308f9cfbbaf..78d71095339 100644 --- a/git-protocol/src/fetch/handshake.rs +++ b/git-protocol/src/fetch/handshake.rs @@ -47,8 +47,6 @@ pub(crate) mod function { /// turns out to be required. `extra_parameters` are the parameters `(name, optional value)` to add to the handshake, /// each time it is performed in case authentication is required. /// `progress` is used to inform about what's currently happening. - /// - /// Note that this function never terminates an existing connection on error as it is assumed to be mutually invalid. #[maybe_async] pub async fn handshake( mut transport: T, diff --git a/git-protocol/src/fetch/refs/function.rs b/git-protocol/src/fetch/refs/function.rs index 2d11831ccab..e4d2ad7df95 100644 --- a/git-protocol/src/fetch/refs/function.rs +++ b/git-protocol/src/fetch/refs/function.rs @@ -11,8 +11,6 @@ use crate::fetch::{indicate_end_of_interaction, refs::from_v2_refs, Command, LsR /// Invoke an ls-refs command on `transport` (assuming `protocol_version` 2 or panic), which requires a prior handshake that yielded /// server `capabilities`. `prepare_ls_refs(capabilities, arguments, features)` can be used to alter the _ls-refs_. `progress` is used to provide feedback. -/// -/// Note that this function can be assumed to terminate the connection (as indicated to the server) if `prepare_ls_refs` has an error. #[maybe_async] pub async fn refs( mut transport: impl Transport, diff --git a/git-protocol/src/fetch_fn.rs b/git-protocol/src/fetch_fn.rs index 44ee175ed98..e19d78deca8 100644 --- a/git-protocol/src/fetch_fn.rs +++ b/git-protocol/src/fetch_fn.rs @@ -1,9 +1,7 @@ use git_features::progress::Progress; use git_transport::client; -use git_transport::client::Capabilities; use maybe_async::maybe_async; -use crate::fetch::{DelegateBlocking, Ref}; use crate::{ credentials, fetch::{handshake, indicate_end_of_interaction, Action, Arguments, Command, Delegate, Error, Response}, @@ -55,7 +53,7 @@ pub async fn fetch( ) -> Result<(), Error> where F: FnMut(credentials::helper::Action) -> credentials::protocol::Result, - D: Delegate + DelegateBlocking, + D: Delegate, T: client::Transport, { let handshake::Outcome { @@ -143,94 +141,6 @@ where Ok(()) } -/// Fetch the pack after a `handshake` has been performed on `transport` and `refs` have been obtained. -/// -/// `negotiate` maybe called repeatedly, after `prepare_fetch` was called once, to setup the haves and wants to fetch. -/// The `delegate` will receive a reader with pack data when done, and `progress` sees all remote end progress messages. -#[maybe_async] -pub async fn fetch_pack( - handshake::Outcome { - server_protocol_version: protocol_version, - refs: _, - capabilities, - }: &handshake::Outcome, - refs: &[Ref], - mut transport: T, - mut prepare_fetch: impl FnMut( - git_transport::Protocol, - &Capabilities, - &mut Vec<(&str, Option<&str>)>, - &[Ref], - ) -> std::io::Result, - mut negotiate: impl FnMut(&[Ref], &mut Arguments, Option<&Response>) -> std::io::Result, - mut delegate: D, - mut progress: impl Progress, - fetch_mode: FetchConnection, -) -> Result<(), Error> -where - F: FnMut(credentials::helper::Action) -> credentials::protocol::Result, - D: Delegate, - T: client::Transport, -{ - let fetch = Command::Fetch; - let mut fetch_features = fetch.default_features(*protocol_version, capabilities); - match prepare_fetch(*protocol_version, capabilities, &mut fetch_features, refs) { - Ok(Action::Cancel) => { - return if matches!(protocol_version, git_transport::Protocol::V1) - || matches!(fetch_mode, FetchConnection::TerminateOnSuccessfulCompletion) - { - indicate_end_of_interaction(transport).await.map_err(Into::into) - } else { - Ok(()) - }; - } - Ok(Action::Continue) => { - fetch.validate_argument_prefixes_or_panic(*protocol_version, capabilities, &[], &fetch_features); - } - Err(err) => { - indicate_end_of_interaction(transport).await?; - return Err(err.into()); - } - } - - Response::check_required_features(*protocol_version, &fetch_features)?; - let sideband_all = fetch_features.iter().any(|(n, _)| *n == "sideband-all"); - let mut arguments = Arguments::new(*protocol_version, fetch_features); - let mut previous_response = None::; - let mut round = 1; - 'negotiation: loop { - progress.step(); - progress.set_name(format!("negotiate (round {})", round)); - round += 1; - let action = negotiate(refs, &mut arguments, previous_response.as_ref())?; - let mut reader = arguments.send(&mut transport, action == Action::Cancel).await?; - if sideband_all { - setup_remote_progress(&mut progress, &mut reader); - } - let response = Response::from_line_reader(*protocol_version, &mut reader).await?; - previous_response = if response.has_pack() { - progress.step(); - progress.set_name("receiving pack"); - if !sideband_all { - setup_remote_progress(&mut progress, &mut reader); - } - delegate.receive_pack(reader, progress, refs, &response).await?; - break 'negotiation; - } else { - match action { - Action::Cancel => break 'negotiation, - Action::Continue => Some(response), - } - } - } - if matches!(protocol_version, git_transport::Protocol::V2) - && matches!(fetch_mode, FetchConnection::TerminateOnSuccessfulCompletion) - { - indicate_end_of_interaction(transport).await?; - } - Ok(()) -} - fn setup_remote_progress( progress: &mut impl Progress, reader: &mut Box, diff --git a/git-protocol/src/lib.rs b/git-protocol/src/lib.rs index 279a02458bb..12c69e87471 100644 --- a/git-protocol/src/lib.rs +++ b/git-protocol/src/lib.rs @@ -26,7 +26,7 @@ pub mod fetch; #[cfg(any(feature = "blocking-client", feature = "async-client"))] mod fetch_fn; #[cfg(any(feature = "blocking-client", feature = "async-client"))] -pub use fetch_fn::{fetch, fetch_pack, FetchConnection}; +pub use fetch_fn::{fetch, FetchConnection}; mod remote_progress; pub use remote_progress::RemoteProgress; From f0f4db6fcb61a5a93786c74c7997657b2fc4f233 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 22 Sep 2022 16:28:32 +0800 Subject: [PATCH 019/113] sketch of 'Prepare' struct to configure fetch after ref-map was obtained. (#450) --- git-repository/src/remote/connection/fetch.rs | 18 +++++++++++++----- git-repository/src/remote/mod.rs | 2 +- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/git-repository/src/remote/connection/fetch.rs b/git-repository/src/remote/connection/fetch.rs index d723c917a67..801e1f99e4e 100644 --- a/git-repository/src/remote/connection/fetch.rs +++ b/git-repository/src/remote/connection/fetch.rs @@ -1,17 +1,25 @@ -use crate::remote::Connection; +use crate::remote::fetch::RefMap; +use crate::remote::{ref_map, Connection}; use crate::Progress; use git_protocol::transport::client::Transport; -#[allow(missing_docs)] -pub struct Options {} - impl<'remote, 'repo, T, P> Connection<'remote, 'repo, T, P> where T: Transport, P: Progress, { + /// Perform a handshake with the remote and obtain a ref-map with `options`, and from there one + /// Note that at this point, the `transport` should be configured using the [`transport_mut()`][Self::transport_mut()] + /// method, as it will be consumed here. #[allow(missing_docs)] - pub fn prepare_fetch(self) -> ! { + pub fn prepare_fetch(self, _options: ref_map::Options) -> Result, ref_map::Error> { todo!() } } + +/// A structure to hold the result of the handshake with the remote and configure the upcoming fetch operation. +#[allow(dead_code)] +pub struct Prepare<'remote, 'repo, T, P> { + con: Connection<'remote, 'repo, T, P>, + ref_map: RefMap<'remote>, +} diff --git a/git-repository/src/remote/mod.rs b/git-repository/src/remote/mod.rs index 327c359fb23..1635f23ad80 100644 --- a/git-repository/src/remote/mod.rs +++ b/git-repository/src/remote/mod.rs @@ -66,7 +66,7 @@ pub mod fetch { } #[cfg(feature = "blocking-network-client")] - pub use super::connection::fetch::Options; + pub use super::connection::fetch::Prepare; } /// From 78ad3df64f2c016ba17b158bd9ab1d2341aab399 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 22 Sep 2022 17:34:55 +0800 Subject: [PATCH 020/113] feat!: add `fetch::Transport::configure` to generically configure any transport. (#450) --- git-repository/tests/remote/fetch.rs | 0 git-transport/src/client/blocking_io/file.rs | 5 +++++ .../src/client/blocking_io/http/curl/mod.rs | 5 +++++ git-transport/src/client/blocking_io/http/mod.rs | 4 ++++ git-transport/src/client/blocking_io/http/traits.rs | 5 +++++ git-transport/src/client/git/blocking_io.rs | 5 +++++ git-transport/src/client/traits.rs | 13 +++++++++++++ 7 files changed, 37 insertions(+) create mode 100644 git-repository/tests/remote/fetch.rs diff --git a/git-repository/tests/remote/fetch.rs b/git-repository/tests/remote/fetch.rs new file mode 100644 index 00000000000..e69de29bb2d diff --git a/git-transport/src/client/blocking_io/file.rs b/git-transport/src/client/blocking_io/file.rs index 0e23075acd2..a60351176cd 100644 --- a/git-transport/src/client/blocking_io/file.rs +++ b/git-transport/src/client/blocking_io/file.rs @@ -1,3 +1,4 @@ +use std::error::Error; use std::process::{self, Command, Stdio}; use bstr::{BString, ByteSlice}; @@ -101,6 +102,10 @@ impl client::TransportWithoutIO for SpawnProcessOnDemand { fn connection_persists_across_multiple_requests(&self) -> bool { true } + + fn configure(&mut self, _config: &[u8]) -> Result<(), Box> { + Ok(()) + } } impl client::Transport for SpawnProcessOnDemand { diff --git a/git-transport/src/client/blocking_io/http/curl/mod.rs b/git-transport/src/client/blocking_io/http/curl/mod.rs index 06085bf5d68..52390773184 100644 --- a/git-transport/src/client/blocking_io/http/curl/mod.rs +++ b/git-transport/src/client/blocking_io/http/curl/mod.rs @@ -1,3 +1,4 @@ +use std::error::Error; use std::{ sync::mpsc::{Receiver, SyncSender}, thread, @@ -100,4 +101,8 @@ impl crate::client::http::Http for Curl { ) -> Result, http::Error> { self.make_request(url, headers, true) } + + fn configure(&mut self, _config: &[u8]) -> Result<(), Box> { + Ok(()) + } } diff --git a/git-transport/src/client/blocking_io/http/mod.rs b/git-transport/src/client/blocking_io/http/mod.rs index e97e6d5d319..37d810a50cf 100644 --- a/git-transport/src/client/blocking_io/http/mod.rs +++ b/git-transport/src/client/blocking_io/http/mod.rs @@ -158,6 +158,10 @@ impl client::TransportWithoutIO for Transport { fn connection_persists_across_multiple_requests(&self) -> bool { false } + + fn configure(&mut self, config: &[u8]) -> Result<(), Box> { + self.http.configure(config) + } } impl client::Transport for Transport { diff --git a/git-transport/src/client/blocking_io/http/traits.rs b/git-transport/src/client/blocking_io/http/traits.rs index be39502dab3..10d39442ebe 100644 --- a/git-transport/src/client/blocking_io/http/traits.rs +++ b/git-transport/src/client/blocking_io/http/traits.rs @@ -67,4 +67,9 @@ pub trait Http { url: &str, headers: impl IntoIterator>, ) -> Result, Error>; + + /// Pass `config` which can deserialize in the implementation's configuration, as documented separately. + /// + /// The caller must know how that `config` data looks like for the intended implementation. + fn configure(&mut self, config: &[u8]) -> Result<(), Box>; } diff --git a/git-transport/src/client/git/blocking_io.rs b/git-transport/src/client/git/blocking_io.rs index 240f00c9903..84b8afa89a0 100644 --- a/git-transport/src/client/git/blocking_io.rs +++ b/git-transport/src/client/git/blocking_io.rs @@ -1,3 +1,4 @@ +use std::error::Error; use std::io::Write; use bstr::BString; @@ -52,6 +53,10 @@ where fn connection_persists_across_multiple_requests(&self) -> bool { true } + + fn configure(&mut self, _config: &[u8]) -> Result<(), Box> { + Ok(()) + } } impl client::Transport for git::Connection diff --git a/git-transport/src/client/traits.rs b/git-transport/src/client/traits.rs index f57b2fe2d09..c87ddafa22d 100644 --- a/git-transport/src/client/traits.rs +++ b/git-transport/src/client/traits.rs @@ -46,6 +46,11 @@ pub trait TransportWithoutIO { /// of the fetch negotiation or that the end of interaction (i.e. no further request will be made) has to be indicated /// to the server for most graceful termination of the connection. fn connection_persists_across_multiple_requests(&self) -> bool; + + /// Pass `config` which can deserialize in the implementation's configuration, as documented separately. + /// + /// The caller must know how that `config` data looks like for the intended implementation. + fn configure(&mut self, config: &[u8]) -> Result<(), Box>; } // Would be nice if the box implementation could auto-forward to all implemented traits. @@ -70,6 +75,10 @@ impl TransportWithoutIO for Box { fn connection_persists_across_multiple_requests(&self) -> bool { self.deref().connection_persists_across_multiple_requests() } + + fn configure(&mut self, config: &[u8]) -> Result<(), Box> { + self.deref_mut().configure(config) + } } impl TransportWithoutIO for &mut T { @@ -93,4 +102,8 @@ impl TransportWithoutIO for &mut T { fn connection_persists_across_multiple_requests(&self) -> bool { self.deref().connection_persists_across_multiple_requests() } + + fn configure(&mut self, config: &[u8]) -> Result<(), Box> { + self.deref_mut().configure(config) + } } From e8428438ff662a9222858d7b9703102eb1bd3cf1 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 22 Sep 2022 18:05:47 +0800 Subject: [PATCH 021/113] adapt to changes in `git-transport` (#450) --- git-protocol/src/fetch/tests/arguments.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/git-protocol/src/fetch/tests/arguments.rs b/git-protocol/src/fetch/tests/arguments.rs index 76c8bd0acaf..5f7009ecbc2 100644 --- a/git-protocol/src/fetch/tests/arguments.rs +++ b/git-protocol/src/fetch/tests/arguments.rs @@ -46,6 +46,10 @@ mod impls { fn connection_persists_across_multiple_requests(&self) -> bool { self.stateful } + + fn configure(&mut self, config: &[u8]) -> Result<(), Box> { + self.inner.configure(config) + } } impl client::Transport for Transport { From d1cb6cc06b9560cb4dd77c18c1bc9afdec21db43 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 22 Sep 2022 18:06:11 +0800 Subject: [PATCH 022/113] fix build (#450) --- gitoxide-core/src/repository/remote.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/gitoxide-core/src/repository/remote.rs b/gitoxide-core/src/repository/remote.rs index a51b5cc1641..39ee2ee0abc 100644 --- a/gitoxide-core/src/repository/remote.rs +++ b/gitoxide-core/src/repository/remote.rs @@ -75,6 +75,7 @@ mod refs_impl { .await? .ref_map(git::remote::ref_map::Options { prefix_from_spec_as_filter_on_remote: !matches!(kind, refs::Kind::Remote), + ..Default::default() }) .await?; From 9b86a1f38b3ea2bd0e639d392849c3660fc08cff Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 22 Sep 2022 18:06:16 +0800 Subject: [PATCH 023/113] remove connect_http() method to encourage changing settings using `transport_mut().configure()`. (#450) This eventually will be used for all kinds of http settings and remote settings, goal being to be zero-conf since everything comes from git-config. --- git-repository/src/config/mod.rs | 3 +- git-repository/src/remote/connect.rs | 24 +++---------- .../src/remote/connection/ref_map.rs | 2 +- git-repository/src/repository/config.rs | 3 ++ git-repository/tests/remote/fetch.rs | 34 +++++++++++++++++++ git-repository/tests/remote/mod.rs | 1 + git-repository/tests/remote/ref_map.rs | 5 ++- 7 files changed, 47 insertions(+), 25 deletions(-) diff --git a/git-repository/src/config/mod.rs b/git-repository/src/config/mod.rs index ef3e30bc439..256ed865043 100644 --- a/git-repository/src/config/mod.rs +++ b/git-repository/src/config/mod.rs @@ -18,7 +18,8 @@ pub struct Snapshot<'repo> { /// Note that the values will only affect this instance of the parent repository, and not other clones that may exist. /// /// Note that these values won't update even if the underlying file(s) change. -// TODO: make it possible to load snapshots with reloading via .config() and write mutated snapshots back to disk. +// TODO: make it possible to load snapshots with reloading via .config() and write mutated snapshots back to disk which should be the way +// to affect all instances of a repo. pub struct SnapshotMut<'repo> { pub(crate) repo: &'repo mut Repository, pub(crate) config: git_config::File<'static>, diff --git a/git-repository/src/remote/connect.rs b/git-repository/src/remote/connect.rs index ffe08f0dc01..f80871c1ca9 100644 --- a/git-repository/src/remote/connect.rs +++ b/git-repository/src/remote/connect.rs @@ -53,6 +53,10 @@ impl<'repo> Remote<'repo> { /// /// Note that the `protocol.version` configuration key affects the transport protocol used to connect, /// with `2` being the default. + /// + /// The transport used for connection can be configured via `transport_mut().configure()` assuming the actually + /// used transport is well known. If that's not the case, the transport can be created by hand and passed to + /// [to_connection_with_transport()][Self::to_connection_with_transport()]. #[cfg(any(feature = "blocking-network-client", feature = "async-network-client-async-std"))] #[git_protocol::maybe_async::maybe_async] pub async fn connect

( @@ -68,26 +72,6 @@ impl<'repo> Remote<'repo> { Ok(self.to_connection_with_transport(transport, progress)) } - /// Connect to the http(s) url suitable for `direction` and return a handle through which operations can be performed. - /// - /// Note that the `protocol.version` configuration key affects the transport protocol used to connect, - /// with `2` being the default, and that the 'dumb'-http protocol isn't supported. - /// - /// Using this method has the advantage of - #[cfg(all(feature = "blocking-http-transport", feature = "blocking-network-client"))] - pub async fn connect_http

( - &self, - direction: crate::remote::Direction, - progress: P, - ) -> Result, P>, Error> - where - P: Progress, - { - let (url, version) = self.sanitized_url_and_version(direction)?; - let transport = http::connect(&url.to_bstring().to_string(), version); - Ok(self.to_connection_with_transport(transport, progress)) - } - /// Produce the sanitized URL and protocol version to use as obtained by querying the repository configuration. /// /// This can be useful when using custom transports to allow additional configuration. diff --git a/git-repository/src/remote/connection/ref_map.rs b/git-repository/src/remote/connection/ref_map.rs index 8207e686d30..98ef3ff331c 100644 --- a/git-repository/src/remote/connection/ref_map.rs +++ b/git-repository/src/remote/connection/ref_map.rs @@ -25,7 +25,7 @@ pub enum Error { #[derive(Debug, Clone)] pub struct Options { /// Use a two-component prefix derived from the ref-spec's source, like `refs/heads/` to let the server pre-filter refs - /// with great potential for savings in traffic and local CPU time. + /// with great potential for savings in traffic and local CPU time. Defaults to `true`. pub prefix_from_spec_as_filter_on_remote: bool, /// Parameters in the form of `(name, optional value)` to add to the handshake. /// diff --git a/git-repository/src/repository/config.rs b/git-repository/src/repository/config.rs index 5f963c114e6..9be15a0a58d 100644 --- a/git-repository/src/repository/config.rs +++ b/git-repository/src/repository/config.rs @@ -10,6 +10,9 @@ impl crate::Repository { } /// Return a mutable snapshot of the configuration as seen upon opening the repository. + /// + /// Note that changes to the configuration are in-memory only and are observed only the this instance + /// of the [`Repository`]. pub fn config_snapshot_mut(&mut self) -> config::SnapshotMut<'_> { let config = self.config.resolved.as_ref().clone(); config::SnapshotMut { repo: self, config } diff --git a/git-repository/tests/remote/fetch.rs b/git-repository/tests/remote/fetch.rs index e69de29bb2d..6b5f08221fe 100644 --- a/git-repository/tests/remote/fetch.rs +++ b/git-repository/tests/remote/fetch.rs @@ -0,0 +1,34 @@ +#[cfg(feature = "blocking-network-client")] +mod blocking_io { + use git_features::progress; + use git_repository as git; + use git_repository::remote::Direction::Fetch; + + use crate::remote; + + #[test] + #[ignore] + fn fetch_pack() -> crate::Result { + for version in [ + None, + Some(git::protocol::transport::Protocol::V2), + Some(git::protocol::transport::Protocol::V1), + ] { + let mut repo = remote::repo("clone"); + if let Some(version) = version { + repo.config_snapshot_mut().set_raw_value( + "protocol", + None, + "version", + (version as u8).to_string().as_str(), + )?; + } + + let remote = repo.find_remote("origin")?; + remote + .connect(Fetch, progress::Discard)? + .prepare_fetch(Default::default())?; + } + Ok(()) + } +} diff --git a/git-repository/tests/remote/mod.rs b/git-repository/tests/remote/mod.rs index 9a73832179d..ea4b5c03e75 100644 --- a/git-repository/tests/remote/mod.rs +++ b/git-repository/tests/remote/mod.rs @@ -17,4 +17,5 @@ pub(crate) fn cow_str(s: &str) -> Cow { } mod connect; +mod fetch; mod ref_map; diff --git a/git-repository/tests/remote/ref_map.rs b/git-repository/tests/remote/ref_map.rs index 6670392e573..dc1194bec41 100644 --- a/git-repository/tests/remote/ref_map.rs +++ b/git-repository/tests/remote/ref_map.rs @@ -24,8 +24,7 @@ mod blocking_io { } let remote = repo.find_remote("origin")?; - let connection = remote.connect(Fetch, progress::Discard)?; - let map = connection.ref_map(Default::default())?; + let map = remote.connect(Fetch, progress::Discard)?.ref_map(Default::default())?; assert_eq!( map.remote_refs.len(), 14, @@ -36,7 +35,7 @@ mod blocking_io { assert_eq!( map.mappings.len(), 11, - "mappings are only a sub-set of all remotes due to refspec matching" + "mappings are only a sub-set of all remotes due to refspec matching, tags are filtered out." ); } Ok(()) From 779eefed97685300f4cd7b09957d3442c96e5b1f Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 22 Sep 2022 18:46:50 +0800 Subject: [PATCH 024/113] Use `&dyn Any` instead of unspecified serialization format, as it's the right way. (#450) This works for all cases were a single program is compiled and there are no dll boundaries, which I think is the way it will be for quite a while. --- git-protocol/src/fetch/tests/arguments.rs | 12 +++++++++++- git-transport/src/client/blocking_io/file.rs | 3 ++- .../src/client/blocking_io/http/curl/mod.rs | 2 +- git-transport/src/client/blocking_io/http/mod.rs | 3 ++- git-transport/src/client/blocking_io/http/traits.rs | 5 ++++- git-transport/src/client/git/async_io.rs | 5 +++++ git-transport/src/client/git/blocking_io.rs | 3 ++- git-transport/src/client/traits.rs | 9 +++++---- 8 files changed, 32 insertions(+), 10 deletions(-) diff --git a/git-protocol/src/fetch/tests/arguments.rs b/git-protocol/src/fetch/tests/arguments.rs index 5f7009ecbc2..bc81f0d6f4b 100644 --- a/git-protocol/src/fetch/tests/arguments.rs +++ b/git-protocol/src/fetch/tests/arguments.rs @@ -47,7 +47,10 @@ mod impls { self.stateful } - fn configure(&mut self, config: &[u8]) -> Result<(), Box> { + fn configure( + &mut self, + config: &dyn std::any::Any, + ) -> Result<(), Box> { self.inner.configure(config) } } @@ -93,6 +96,13 @@ mod impls { fn connection_persists_across_multiple_requests(&self) -> bool { self.stateful } + + fn configure( + &mut self, + config: &dyn std::any::Any, + ) -> Result<(), Box> { + self.inner.configure(config) + } } #[async_trait(?Send)] diff --git a/git-transport/src/client/blocking_io/file.rs b/git-transport/src/client/blocking_io/file.rs index a60351176cd..2ea33b27bda 100644 --- a/git-transport/src/client/blocking_io/file.rs +++ b/git-transport/src/client/blocking_io/file.rs @@ -1,3 +1,4 @@ +use std::any::Any; use std::error::Error; use std::process::{self, Command, Stdio}; @@ -103,7 +104,7 @@ impl client::TransportWithoutIO for SpawnProcessOnDemand { true } - fn configure(&mut self, _config: &[u8]) -> Result<(), Box> { + fn configure(&mut self, _config: &dyn Any) -> Result<(), Box> { Ok(()) } } diff --git a/git-transport/src/client/blocking_io/http/curl/mod.rs b/git-transport/src/client/blocking_io/http/curl/mod.rs index 52390773184..360cbb28ac9 100644 --- a/git-transport/src/client/blocking_io/http/curl/mod.rs +++ b/git-transport/src/client/blocking_io/http/curl/mod.rs @@ -102,7 +102,7 @@ impl crate::client::http::Http for Curl { self.make_request(url, headers, true) } - fn configure(&mut self, _config: &[u8]) -> Result<(), Box> { + fn configure(&mut self, _config: &dyn std::any::Any) -> Result<(), Box> { Ok(()) } } diff --git a/git-transport/src/client/blocking_io/http/mod.rs b/git-transport/src/client/blocking_io/http/mod.rs index 37d810a50cf..b83c77dab7e 100644 --- a/git-transport/src/client/blocking_io/http/mod.rs +++ b/git-transport/src/client/blocking_io/http/mod.rs @@ -1,3 +1,4 @@ +use std::any::Any; use std::{ borrow::Cow, io::{BufRead, Read}, @@ -159,7 +160,7 @@ impl client::TransportWithoutIO for Transport { false } - fn configure(&mut self, config: &[u8]) -> Result<(), Box> { + fn configure(&mut self, config: &dyn Any) -> Result<(), Box> { self.http.configure(config) } } diff --git a/git-transport/src/client/blocking_io/http/traits.rs b/git-transport/src/client/blocking_io/http/traits.rs index 10d39442ebe..d9103658385 100644 --- a/git-transport/src/client/blocking_io/http/traits.rs +++ b/git-transport/src/client/blocking_io/http/traits.rs @@ -71,5 +71,8 @@ pub trait Http { /// Pass `config` which can deserialize in the implementation's configuration, as documented separately. /// /// The caller must know how that `config` data looks like for the intended implementation. - fn configure(&mut self, config: &[u8]) -> Result<(), Box>; + fn configure( + &mut self, + config: &dyn std::any::Any, + ) -> Result<(), Box>; } diff --git a/git-transport/src/client/git/async_io.rs b/git-transport/src/client/git/async_io.rs index 5df5a648f70..bd5b9a14d16 100644 --- a/git-transport/src/client/git/async_io.rs +++ b/git-transport/src/client/git/async_io.rs @@ -3,6 +3,7 @@ use bstr::BString; use futures_io::{AsyncRead, AsyncWrite}; use futures_lite::AsyncWriteExt; use git_packetline::PacketLineRef; +use std::error::Error; use crate::{ client::{self, capabilities, git, Capabilities, SetServiceResponse}, @@ -50,6 +51,10 @@ where fn connection_persists_across_multiple_requests(&self) -> bool { true } + + fn configure(&mut self, _config: &dyn std::any::Any) -> Result<(), Box> { + Ok(()) + } } #[async_trait(?Send)] diff --git a/git-transport/src/client/git/blocking_io.rs b/git-transport/src/client/git/blocking_io.rs index 84b8afa89a0..97b950dcacf 100644 --- a/git-transport/src/client/git/blocking_io.rs +++ b/git-transport/src/client/git/blocking_io.rs @@ -1,3 +1,4 @@ +use std::any::Any; use std::error::Error; use std::io::Write; @@ -54,7 +55,7 @@ where true } - fn configure(&mut self, _config: &[u8]) -> Result<(), Box> { + fn configure(&mut self, _config: &dyn Any) -> Result<(), Box> { Ok(()) } } diff --git a/git-transport/src/client/traits.rs b/git-transport/src/client/traits.rs index c87ddafa22d..2e34bf03ac7 100644 --- a/git-transport/src/client/traits.rs +++ b/git-transport/src/client/traits.rs @@ -1,3 +1,4 @@ +use std::any::Any; use std::ops::{Deref, DerefMut}; #[cfg(any(feature = "blocking-client", feature = "async-client"))] @@ -47,10 +48,10 @@ pub trait TransportWithoutIO { /// to the server for most graceful termination of the connection. fn connection_persists_across_multiple_requests(&self) -> bool; - /// Pass `config` which can deserialize in the implementation's configuration, as documented separately. + /// Pass `config` can be cast and interpreted by the implementation, as documented separately. /// /// The caller must know how that `config` data looks like for the intended implementation. - fn configure(&mut self, config: &[u8]) -> Result<(), Box>; + fn configure(&mut self, config: &dyn Any) -> Result<(), Box>; } // Would be nice if the box implementation could auto-forward to all implemented traits. @@ -76,7 +77,7 @@ impl TransportWithoutIO for Box { self.deref().connection_persists_across_multiple_requests() } - fn configure(&mut self, config: &[u8]) -> Result<(), Box> { + fn configure(&mut self, config: &dyn Any) -> Result<(), Box> { self.deref_mut().configure(config) } } @@ -103,7 +104,7 @@ impl TransportWithoutIO for &mut T { self.deref().connection_persists_across_multiple_requests() } - fn configure(&mut self, config: &[u8]) -> Result<(), Box> { + fn configure(&mut self, config: &dyn Any) -> Result<(), Box> { self.deref_mut().configure(config) } } From 7993f6a4b95a18809e98f34366dc1746b944f8d5 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 22 Sep 2022 18:52:43 +0800 Subject: [PATCH 025/113] fix build (#450) --- git-repository/src/config/cache/init.rs | 2 +- git-repository/src/config/cache/mod.rs | 2 +- git-repository/src/config/mod.rs | 2 +- git-repository/src/remote/connect.rs | 8 +------- git-repository/src/remote/url/mod.rs | 4 ++-- git-repository/src/repository/config.rs | 2 +- 6 files changed, 7 insertions(+), 13 deletions(-) diff --git a/git-repository/src/config/cache/init.rs b/git-repository/src/config/cache/init.rs index 4b5c87fd31a..c9c03d426ff 100644 --- a/git-repository/src/config/cache/init.rs +++ b/git-repository/src/config/cache/init.rs @@ -158,7 +158,7 @@ impl Cache { home_env, personas: Default::default(), url_rewrite: Default::default(), - #[cfg(any(feature = "blocking-network-client", feature = "async-network-client-async-std"))] + #[cfg(any(feature = "blocking-network-client", feature = "async-network-client"))] url_scheme: Default::default(), git_prefix, }) diff --git a/git-repository/src/config/cache/mod.rs b/git-repository/src/config/cache/mod.rs index eeae0c20426..e8cf78ba300 100644 --- a/git-repository/src/config/cache/mod.rs +++ b/git-repository/src/config/cache/mod.rs @@ -24,7 +24,7 @@ impl Cache { .get_or_init(|| remote::url::Rewrite::from_config(&self.resolved, self.filter_config_section)) } - #[cfg(any(feature = "blocking-network-client", feature = "async-network-client-async-std"))] + #[cfg(any(feature = "blocking-network-client", feature = "async-network-client"))] pub(crate) fn url_scheme( &self, ) -> Result<&remote::url::SchemePermission, remote::url::scheme_permission::init::Error> { diff --git a/git-repository/src/config/mod.rs b/git-repository/src/config/mod.rs index 256ed865043..392e4921a0e 100644 --- a/git-repository/src/config/mod.rs +++ b/git-repository/src/config/mod.rs @@ -74,7 +74,7 @@ pub(crate) struct Cache { /// A lazily loaded rewrite list for remote urls pub url_rewrite: OnceCell, /// A lazily loaded mapping to know which url schemes to allow - #[cfg(any(feature = "blocking-network-client", feature = "async-network-client-async-std"))] + #[cfg(any(feature = "blocking-network-client", feature = "async-network-client"))] pub url_scheme: OnceCell, /// The config section filter from the options used to initialize this instance. Keep these in sync! filter_config_section: fn(&git_config::file::Metadata) -> bool, diff --git a/git-repository/src/remote/connect.rs b/git-repository/src/remote/connect.rs index f80871c1ca9..6e18516dde6 100644 --- a/git-repository/src/remote/connect.rs +++ b/git-repository/src/remote/connect.rs @@ -1,8 +1,6 @@ -use git_protocol::transport::client::Transport; - use crate::{remote::Connection, Progress, Remote}; +use git_protocol::transport::client::Transport; -#[cfg(any(feature = "blocking-network-client", feature = "async-network-client-async-std"))] mod error { use crate::{bstr::BString, remote}; @@ -24,12 +22,8 @@ mod error { FileUrl(#[from] git_discover::is_git::Error), } } -#[cfg(any(feature = "blocking-network-client", feature = "async-network-client-async-std"))] pub use error::Error; -#[cfg(all(feature = "blocking-http-transport", feature = "blocking-network-client"))] -use git_protocol::transport::client::http; - /// Establishing connections to remote hosts (without performing a git-handshake). impl<'repo> Remote<'repo> { /// Create a new connection using `transport` to communicate, with `progress` to indicate changes. diff --git a/git-repository/src/remote/url/mod.rs b/git-repository/src/remote/url/mod.rs index 66ec6c66094..7b881581298 100644 --- a/git-repository/src/remote/url/mod.rs +++ b/git-repository/src/remote/url/mod.rs @@ -1,7 +1,7 @@ mod rewrite; /// -#[cfg(any(feature = "blocking-network-client", feature = "async-network-client-async-std"))] +#[cfg(any(feature = "blocking-network-client", feature = "async-network-client"))] pub mod scheme_permission; pub(crate) use rewrite::Rewrite; -#[cfg(any(feature = "blocking-network-client", feature = "async-network-client-async-std"))] +#[cfg(any(feature = "blocking-network-client", feature = "async-network-client"))] pub(crate) use scheme_permission::SchemePermission; diff --git a/git-repository/src/repository/config.rs b/git-repository/src/repository/config.rs index 9be15a0a58d..3de02e53443 100644 --- a/git-repository/src/repository/config.rs +++ b/git-repository/src/repository/config.rs @@ -12,7 +12,7 @@ impl crate::Repository { /// Return a mutable snapshot of the configuration as seen upon opening the repository. /// /// Note that changes to the configuration are in-memory only and are observed only the this instance - /// of the [`Repository`]. + /// of the [`Repository`][crate::Repository]. pub fn config_snapshot_mut(&mut self) -> config::SnapshotMut<'_> { let config = self.config.resolved.as_ref().clone(); config::SnapshotMut { repo: self, config } From f8fb04ad76d282ea3b31cba512f7421f31569e8b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 23 Sep 2022 09:39:28 +0800 Subject: [PATCH 026/113] refactor (#450) --- .../src/remote/connection/access.rs | 48 +++++++++++++++++ git-repository/src/remote/connection/mod.rs | 51 +------------------ 2 files changed, 49 insertions(+), 50 deletions(-) create mode 100644 git-repository/src/remote/connection/access.rs diff --git a/git-repository/src/remote/connection/access.rs b/git-repository/src/remote/connection/access.rs new file mode 100644 index 00000000000..cafb887bb8d --- /dev/null +++ b/git-repository/src/remote/connection/access.rs @@ -0,0 +1,48 @@ +use crate::{ + remote::{connection::CredentialsFn, Connection}, + Remote, +}; + +/// Builder +impl<'a, 'repo, T, P> Connection<'a, 'repo, T, P> { + /// Set a custom credentials callback to provide credentials if the remotes require authentication. + /// + /// Otherwise we will use the git configuration to perform the same task as the `git credential` helper program, + /// which is calling other helper programs in succession while resorting to a prompt to obtain credentials from the + /// user. + /// + /// A custom function may also be used to prevent accessing resources with authentication. + pub fn with_credentials( + mut self, + helper: impl FnMut(git_credentials::helper::Action) -> git_credentials::protocol::Result + 'a, + ) -> Self { + self.credentials = Some(Box::new(helper)); + self + } +} + +/// Access +impl<'a, 'repo, T, P> Connection<'a, 'repo, T, P> { + /// A utility to return a function that will use this repository's configuration to obtain credentials, similar to + /// what `git credential` is doing. + /// + /// It's meant to be used by users of the [`with_credentials()`][Self::with_credentials()] builder to gain access to the + /// default way of handling credentials, which they can call as fallback. + pub fn configured_credentials( + &self, + url: git_url::Url, + ) -> Result, crate::config::credential_helpers::Error> { + let (mut cascade, _action_with_normalized_url, prompt_opts) = + self.remote.repo.config_snapshot().credential_helpers(url)?; + Ok(Box::new(move |action| cascade.invoke(action, prompt_opts.clone())) as CredentialsFn<'_>) + } + /// Drop the transport and additional state to regain the original remote. + pub fn remote(&self) -> &Remote<'repo> { + self.remote + } + + /// Provide a mutable transport to allow configuring it. + pub fn transport_mut(&mut self) -> &mut T { + &mut self.transport + } +} diff --git a/git-repository/src/remote/connection/mod.rs b/git-repository/src/remote/connection/mod.rs index 98bcbdc7aa7..44881ae61b8 100644 --- a/git-repository/src/remote/connection/mod.rs +++ b/git-repository/src/remote/connection/mod.rs @@ -19,56 +19,7 @@ pub struct Connection<'a, 'repo, T, P> { pub(crate) progress: P, } -mod access { - use crate::{ - remote::{connection::CredentialsFn, Connection}, - Remote, - }; - - /// Builder - impl<'a, 'repo, T, P> Connection<'a, 'repo, T, P> { - /// Set a custom credentials callback to provide credentials if the remotes require authentication. - /// - /// Otherwise we will use the git configuration to perform the same task as the `git credential` helper program, - /// which is calling other helper programs in succession while resorting to a prompt to obtain credentials from the - /// user. - /// - /// A custom function may also be used to prevent accessing resources with authentication. - pub fn with_credentials( - mut self, - helper: impl FnMut(git_credentials::helper::Action) -> git_credentials::protocol::Result + 'a, - ) -> Self { - self.credentials = Some(Box::new(helper)); - self - } - } - - /// Access - impl<'a, 'repo, T, P> Connection<'a, 'repo, T, P> { - /// A utility to return a function that will use this repository's configuration to obtain credentials, similar to - /// what `git credential` is doing. - /// - /// It's meant to be used by users of the [`with_credentials()`][Self::with_credentials()] builder to gain access to the - /// default way of handling credentials, which they can call as fallback. - pub fn configured_credentials( - &self, - url: git_url::Url, - ) -> Result, crate::config::credential_helpers::Error> { - let (mut cascade, _action_with_normalized_url, prompt_opts) = - self.remote.repo.config_snapshot().credential_helpers(url)?; - Ok(Box::new(move |action| cascade.invoke(action, prompt_opts.clone())) as CredentialsFn<'_>) - } - /// Drop the transport and additional state to regain the original remote. - pub fn remote(&self) -> &Remote<'repo> { - self.remote - } - - /// Provide a mutable transport to allow configuring it. - pub fn transport_mut(&mut self) -> &mut T { - &mut self.transport - } - } -} +mod access; /// pub mod ref_map; From 4b1e3b3d91c51da3dbea9191e60f959a1266cf47 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 23 Sep 2022 10:23:47 +0800 Subject: [PATCH 027/113] feat: add `Repository::find_default_remote()` which works on detached heads as well. (#450) --- git-repository/src/head/mod.rs | 13 ++++++++----- git-repository/src/reference/remote.rs | 2 ++ git-repository/src/repository/remote.rs | 11 +++++++++++ .../tests/fixtures/make_remote_repos.sh | 4 ++++ git-repository/tests/head/mod.rs | 12 +++++++++++- git-repository/tests/repository/remote.rs | 19 +++++++++++++++++++ 6 files changed, 55 insertions(+), 6 deletions(-) diff --git a/git-repository/src/head/mod.rs b/git-repository/src/head/mod.rs index 87ddd8ce310..349e8b93a6c 100644 --- a/git-repository/src/head/mod.rs +++ b/git-repository/src/head/mod.rs @@ -87,19 +87,22 @@ mod remote { /// Remote impl<'repo> Head<'repo> { /// Return the remote with which the currently checked our reference can be handled as configured by `branch..remote|pushRemote` - /// or fall back to the non-branch specific remote configuration. `None` is returned if the head is detached or unborn. + /// or fall back to the non-branch specific remote configuration. `None` is returned if the head is detached or unborn, so there is + /// no branch specific remote. /// /// This is equivalent to calling [`Reference::remote(…)`][crate::Reference::remote()] and /// [`Repository::remote_default_name()`][crate::Repository::remote_default_name()] in order. + /// + /// Combine it with [`find_default_remote()`][crate::Repository::find_default_remote()] as fallback to handle detached heads, + /// i.e. obtain a remote even in case of detached heads. pub fn into_remote( self, direction: remote::Direction, ) -> Option, remote::find::existing::Error>> { let repo = self.repo; - self.try_into_referent()?.remote(direction).or_else(|| { - repo.remote_default_name(direction) - .map(|name| repo.find_remote(name.as_ref())) - }) + self.try_into_referent()? + .remote(direction) + .or_else(|| repo.find_default_remote(direction)) } } } diff --git a/git-repository/src/reference/remote.rs b/git-repository/src/reference/remote.rs index c6a4f6193d5..79600bd5238 100644 --- a/git-repository/src/reference/remote.rs +++ b/git-repository/src/reference/remote.rs @@ -72,6 +72,8 @@ impl<'repo> Reference<'repo> { /// Like [`remote_name(…)`][Self::remote_name()], but configures the returned `Remote` with additional information like /// /// - `branch..merge` to know which branch on the remote side corresponds to this one for merging when pulling. + /// + /// It also handles if the remote is a configured URL, which has no name. pub fn remote( &self, direction: remote::Direction, diff --git a/git-repository/src/repository/remote.rs b/git-repository/src/repository/remote.rs index e9d53e1ca45..c23fbb10b68 100644 --- a/git-repository/src/repository/remote.rs +++ b/git-repository/src/repository/remote.rs @@ -32,6 +32,17 @@ impl crate::Repository { .ok_or_else(|| find::existing::Error::NotFound { name: name.into() })??) } + /// Find the default remote as configured, or `None` if no such configuration could be found. + /// + /// See [remote_default_name()][Self::remote_default_name()] for more information on the `direction` parameter. + pub fn find_default_remote( + &self, + direction: remote::Direction, + ) -> Option, find::existing::Error>> { + self.remote_default_name(direction) + .map(|name| self.find_remote(name.as_ref())) + } + /// Find the remote with the given `name` or return `None` if it doesn't exist, for the purpose of fetching or pushing /// data to a remote. /// diff --git a/git-repository/tests/fixtures/make_remote_repos.sh b/git-repository/tests/fixtures/make_remote_repos.sh index 7331e48ae9f..2f05bac7822 100644 --- a/git-repository/tests/fixtures/make_remote_repos.sh +++ b/git-repository/tests/fixtures/make_remote_repos.sh @@ -274,3 +274,7 @@ git clone --shared base credential-helpers baseline "git://host.org" ) +git clone --shared base detached-head +(cd detached-head + git checkout @~1 +) diff --git a/git-repository/tests/head/mod.rs b/git-repository/tests/head/mod.rs index 5940a9c4871..c8e2fa6a940 100644 --- a/git-repository/tests/head/mod.rs +++ b/git-repository/tests/head/mod.rs @@ -1,4 +1,4 @@ -mod remote { +mod into_remote { use git_repository as git; use crate::remote; @@ -12,4 +12,14 @@ mod remote { ); Ok(()) } + + #[test] + fn detached_is_none() -> crate::Result { + let repo = remote::repo("detached-head"); + assert_eq!( + repo.head()?.into_remote(git::remote::Direction::Fetch).transpose()?, + None + ); + Ok(()) + } } diff --git a/git-repository/tests/repository/remote.rs b/git-repository/tests/repository/remote.rs index d144b6b5fd2..d029b7074ec 100644 --- a/git-repository/tests/repository/remote.rs +++ b/git-repository/tests/repository/remote.rs @@ -271,3 +271,22 @@ mod find_remote { .to_string() } } + +mod find_default_remote { + use crate::remote; + use git_repository as git; + + #[test] + fn works_on_detached_heads() -> crate::Result { + let repo = remote::repo("detached-head"); + assert_eq!( + repo.find_default_remote(git::remote::Direction::Fetch) + .transpose()? + .expect("present") + .name() + .expect("always named"), + "origin" + ); + Ok(()) + } +} From 249c54ef237c8147dce4cd999ccd4ddba4775150 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 23 Sep 2022 10:59:05 +0800 Subject: [PATCH 028/113] allow stopping fetches after preparing it (#450) --- git-repository/src/remote/connection/fetch.rs | 37 ++++++++++++++++--- .../src/remote/connection/ref_map.rs | 2 +- git-repository/tests/remote/fetch.rs | 10 +++-- 3 files changed, 38 insertions(+), 11 deletions(-) diff --git a/git-repository/src/remote/connection/fetch.rs b/git-repository/src/remote/connection/fetch.rs index 801e1f99e4e..c5d1ee17482 100644 --- a/git-repository/src/remote/connection/fetch.rs +++ b/git-repository/src/remote/connection/fetch.rs @@ -9,17 +9,42 @@ where P: Progress, { /// Perform a handshake with the remote and obtain a ref-map with `options`, and from there one - /// Note that at this point, the `transport` should be configured using the [`transport_mut()`][Self::transport_mut()] + /// Note that at this point, the `transport` should already be configured using the [`transport_mut()`][Self::transport_mut()] /// method, as it will be consumed here. - #[allow(missing_docs)] - pub fn prepare_fetch(self, _options: ref_map::Options) -> Result, ref_map::Error> { - todo!() + /// + /// From there additional properties of the fetch can be adjusted to override the defaults that are configured via git-config. + /// + /// # Blocking Only + /// + /// Note that this implementation is currently limited to blocking mode as it relies on Drop semantics to close the connection + /// should the fetch not be performed. Furthermore, there the code doing the fetch is inherently blocking so there is no benefit. + /// It's best to unblock it by placing it into its own thread or offload it should usage in an async context be required. + pub fn prepare_fetch(mut self, options: ref_map::Options) -> Result, ref_map::Error> { + let ref_map = self.ref_map_inner(options)?; + Ok(Prepare { + con: Some(self), + ref_map, + }) } } /// A structure to hold the result of the handshake with the remote and configure the upcoming fetch operation. #[allow(dead_code)] -pub struct Prepare<'remote, 'repo, T, P> { - con: Connection<'remote, 'repo, T, P>, +pub struct Prepare<'remote, 'repo, T, P> +where + T: Transport, +{ + con: Option>, ref_map: RefMap<'remote>, } + +impl<'remote, 'repo, T, P> Drop for Prepare<'remote, 'repo, T, P> +where + T: Transport, +{ + fn drop(&mut self) { + if let Some(mut con) = self.con.take() { + git_protocol::fetch::indicate_end_of_interaction(&mut con.transport).ok(); + } + } +} diff --git a/git-repository/src/remote/connection/ref_map.rs b/git-repository/src/remote/connection/ref_map.rs index 98ef3ff331c..3e0a9ffb2d9 100644 --- a/git-repository/src/remote/connection/ref_map.rs +++ b/git-repository/src/remote/connection/ref_map.rs @@ -69,7 +69,7 @@ where } #[git_protocol::maybe_async::maybe_async] - async fn ref_map_inner( + pub(crate) async fn ref_map_inner( &mut self, Options { prefix_from_spec_as_filter_on_remote, diff --git a/git-repository/tests/remote/fetch.rs b/git-repository/tests/remote/fetch.rs index 6b5f08221fe..3150fcb04a1 100644 --- a/git-repository/tests/remote/fetch.rs +++ b/git-repository/tests/remote/fetch.rs @@ -7,7 +7,6 @@ mod blocking_io { use crate::remote; #[test] - #[ignore] fn fetch_pack() -> crate::Result { for version in [ None, @@ -25,9 +24,12 @@ mod blocking_io { } let remote = repo.find_remote("origin")?; - remote - .connect(Fetch, progress::Discard)? - .prepare_fetch(Default::default())?; + { + remote + .connect(Fetch, progress::Discard)? + .prepare_fetch(Default::default())?; + // early drops are fine and won't block. + } } Ok(()) } From 5c2442510a8dbb94c9f96951922d9f9cd1784cc1 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 23 Sep 2022 11:41:51 +0800 Subject: [PATCH 029/113] be a bit clearer on how configuration should be done. (#450) --- DEVELOPMENT.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 66b00b0f37a..bf1e5c1e121 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -18,6 +18,18 @@ * We `thiserror` generally. * Adhere to the [stability guide](https://github.com/Byron/gitoxide/blob/main/STABILITY.md) +## Configuration and overrides + +As a general rule, respect and implement all applicable [git-config](https://git-scm.com/docs/git-config) by default, but allow the +caller to set overrides. How overrides work depends on the goals of the particular API so it can be done on the main call path, +forcing a choice, or more typically, as a side-lane where overrides can be done on demand. + +Note that it should be possible to obtain the current configuration for modification by the user for selective overrides, either +by calling methods or by obtaining a data structure that can be set as a whole using a `get -> modify -> set` cycle. + +Parameters which are not available in git or specific to `gitoxide` or the needs of the caller can be passed as parameters or via +`Options` or `Context` structures as needed. + ## General * **async** From 67801a344a4fc6d7c171d93277635bdf84e6c15a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 23 Sep 2022 12:01:39 +0800 Subject: [PATCH 030/113] sketch the receive() method to finally receive a pack. (#450) --- git-repository/src/remote/connect.rs | 2 +- .../src/remote/connection/access.rs | 12 +++++----- git-repository/src/remote/connection/fetch.rs | 22 +++++++++++++++++++ git-repository/src/remote/connection/mod.rs | 6 ++--- .../src/remote/connection/ref_map.rs | 2 +- 5 files changed, 33 insertions(+), 11 deletions(-) diff --git a/git-repository/src/remote/connect.rs b/git-repository/src/remote/connect.rs index 6e18516dde6..55cc129c6dd 100644 --- a/git-repository/src/remote/connect.rs +++ b/git-repository/src/remote/connect.rs @@ -37,7 +37,7 @@ impl<'repo> Remote<'repo> { { Connection { remote: self, - credentials: None, + authenticate: None, transport, progress, } diff --git a/git-repository/src/remote/connection/access.rs b/git-repository/src/remote/connection/access.rs index cafb887bb8d..dbdbc9204ff 100644 --- a/git-repository/src/remote/connection/access.rs +++ b/git-repository/src/remote/connection/access.rs @@ -1,5 +1,5 @@ use crate::{ - remote::{connection::CredentialsFn, Connection}, + remote::{connection::AuthenticateFn, Connection}, Remote, }; @@ -16,7 +16,7 @@ impl<'a, 'repo, T, P> Connection<'a, 'repo, T, P> { mut self, helper: impl FnMut(git_credentials::helper::Action) -> git_credentials::protocol::Result + 'a, ) -> Self { - self.credentials = Some(Box::new(helper)); + self.authenticate = Some(Box::new(helper)); self } } @@ -31,17 +31,17 @@ impl<'a, 'repo, T, P> Connection<'a, 'repo, T, P> { pub fn configured_credentials( &self, url: git_url::Url, - ) -> Result, crate::config::credential_helpers::Error> { + ) -> Result, crate::config::credential_helpers::Error> { let (mut cascade, _action_with_normalized_url, prompt_opts) = self.remote.repo.config_snapshot().credential_helpers(url)?; - Ok(Box::new(move |action| cascade.invoke(action, prompt_opts.clone())) as CredentialsFn<'_>) + Ok(Box::new(move |action| cascade.invoke(action, prompt_opts.clone())) as AuthenticateFn<'_>) } - /// Drop the transport and additional state to regain the original remote. + /// Return the underlying remote that instantiate this connection. pub fn remote(&self) -> &Remote<'repo> { self.remote } - /// Provide a mutable transport to allow configuring it. + /// Provide a mutable transport to allow configuring it with [`configure()`][git_protocol::transport::client::TransportWithoutIO::configure()] pub fn transport_mut(&mut self) -> &mut T { &mut self.transport } diff --git a/git-repository/src/remote/connection/fetch.rs b/git-repository/src/remote/connection/fetch.rs index c5d1ee17482..43130ceef36 100644 --- a/git-repository/src/remote/connection/fetch.rs +++ b/git-repository/src/remote/connection/fetch.rs @@ -2,6 +2,15 @@ use crate::remote::fetch::RefMap; use crate::remote::{ref_map, Connection}; use crate::Progress; use git_protocol::transport::client::Transport; +use std::sync::atomic::AtomicBool; + +mod error { + /// The error returned by [`receive()`](super::Prepare::receive()). + #[derive(Debug, thiserror::Error)] + #[error("TBD")] + pub enum Error {} +} +pub use error::Error; impl<'remote, 'repo, T, P> Connection<'remote, 'repo, T, P> where @@ -28,6 +37,19 @@ where } } +impl<'remote, 'repo, T, P> Prepare<'remote, 'repo, T, P> +where + T: Transport, + P: Progress, +{ + /// receive the pack and perform the operation as configured by git via `git-config` or overridden by various builder methods. + pub fn receive(mut self, _should_interrupt: &AtomicBool) -> Result<(), Error> { + let mut con = self.con.take().expect("receive() can only be called once"); + git_protocol::fetch::indicate_end_of_interaction(&mut con.transport).ok(); + todo!() + } +} + /// A structure to hold the result of the handshake with the remote and configure the upcoming fetch operation. #[allow(dead_code)] pub struct Prepare<'remote, 'repo, T, P> diff --git a/git-repository/src/remote/connection/mod.rs b/git-repository/src/remote/connection/mod.rs index 44881ae61b8..8c875f9259f 100644 --- a/git-repository/src/remote/connection/mod.rs +++ b/git-repository/src/remote/connection/mod.rs @@ -5,8 +5,8 @@ pub(crate) struct HandshakeWithRefs { refs: Vec, } -/// A function that performs a given credential action. -pub type CredentialsFn<'a> = Box git_credentials::protocol::Result + 'a>; +/// A function that performs a given credential action, trying to obtain credentials for an operation that needs it. +pub type AuthenticateFn<'a> = Box git_credentials::protocol::Result + 'a>; /// A type to represent an ongoing connection to a remote host, typically with the connection already established. /// @@ -14,7 +14,7 @@ pub type CredentialsFn<'a> = Box g /// much like a remote procedure call. pub struct Connection<'a, 'repo, T, P> { pub(crate) remote: &'a Remote<'repo>, - pub(crate) credentials: Option>, + pub(crate) authenticate: Option>, pub(crate) transport: T, pub(crate) progress: P, } diff --git a/git-repository/src/remote/connection/ref_map.rs b/git-repository/src/remote/connection/ref_map.rs index 3e0a9ffb2d9..ed37f38a0fe 100644 --- a/git-repository/src/remote/connection/ref_map.rs +++ b/git-repository/src/remote/connection/ref_map.rs @@ -121,7 +121,7 @@ where extra_parameters: Vec<(String, Option)>, ) -> Result { let mut credentials_storage; - let authenticate = match self.credentials.as_mut() { + let authenticate = match self.authenticate.as_mut() { Some(f) => f, None => { let url = self From d2bea003230078ffb4e6cd80d1b01c3995435a34 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 23 Sep 2022 13:04:46 +0800 Subject: [PATCH 031/113] feat: add `config::SnapshotMut::forget()` to forget all changes before applying them. (#450) The documentation was update to make clear when the changes are applied. --- DEVELOPMENT.md | 3 +++ git-repository/src/config/mod.rs | 2 ++ git-repository/src/config/snapshot/access.rs | 9 +++++++++ git-repository/src/repository/config.rs | 3 ++- 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index bf1e5c1e121..2a11a4e4f26 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -27,6 +27,9 @@ forcing a choice, or more typically, as a side-lane where overrides can be done Note that it should be possible to obtain the current configuration for modification by the user for selective overrides, either by calling methods or by obtaining a data structure that can be set as a whole using a `get -> modify -> set` cycle. +Note that without any of that, one should document that with `config_snapshot_mut()` any of the relevant configuration can be +changed in memory before invoking a method in order to affect it. + Parameters which are not available in git or specific to `gitoxide` or the needs of the caller can be passed as parameters or via `Options` or `Context` structures as needed. diff --git a/git-repository/src/config/mod.rs b/git-repository/src/config/mod.rs index 392e4921a0e..49f6df8363d 100644 --- a/git-repository/src/config/mod.rs +++ b/git-repository/src/config/mod.rs @@ -18,6 +18,8 @@ pub struct Snapshot<'repo> { /// Note that the values will only affect this instance of the parent repository, and not other clones that may exist. /// /// Note that these values won't update even if the underlying file(s) change. +/// +/// Use [`forget()`][Self::forget()] to not apply any of the changes. // TODO: make it possible to load snapshots with reloading via .config() and write mutated snapshots back to disk which should be the way // to affect all instances of a repo. pub struct SnapshotMut<'repo> { diff --git a/git-repository/src/config/snapshot/access.rs b/git-repository/src/config/snapshot/access.rs index c386436f29f..cb978a5e701 100644 --- a/git-repository/src/config/snapshot/access.rs +++ b/git-repository/src/config/snapshot/access.rs @@ -1,5 +1,6 @@ use std::borrow::Cow; +use crate::config::SnapshotMut; use crate::{ bstr::BStr, config::{cache::interpolate_context, Snapshot}, @@ -90,3 +91,11 @@ impl<'repo> Snapshot<'repo> { &self.repo.config.resolved } } + +/// Utilities +impl<'repo> SnapshotMut<'repo> { + /// Don't apply any of the changes after consuming this instance, effectively forgetting them. + pub fn forget(mut self) { + std::mem::take(&mut self.config); + } +} diff --git a/git-repository/src/repository/config.rs b/git-repository/src/repository/config.rs index 3de02e53443..860136bc969 100644 --- a/git-repository/src/repository/config.rs +++ b/git-repository/src/repository/config.rs @@ -9,7 +9,8 @@ impl crate::Repository { config::Snapshot { repo: self } } - /// Return a mutable snapshot of the configuration as seen upon opening the repository. + /// Return a mutable snapshot of the configuration as seen upon opening the repository, starting a transaction. + /// When the returned instance is dropped, it is applied in full, even if the reason for the drop is an error. /// /// Note that changes to the configuration are in-memory only and are observed only the this instance /// of the [`Repository`][crate::Repository]. From 4367994a8a7476eb44e1309e833e345fdb78f246 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 23 Sep 2022 13:15:51 +0800 Subject: [PATCH 032/113] feat: add `config::SnapshotMut::commit()` to make clear it's transactional. (#450) --- git-repository/src/config/snapshot/access.rs | 5 +++++ git-repository/src/config/snapshot/apply_cli_overrides.rs | 7 +++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/git-repository/src/config/snapshot/access.rs b/git-repository/src/config/snapshot/access.rs index cb978a5e701..b9cf25d5db8 100644 --- a/git-repository/src/config/snapshot/access.rs +++ b/git-repository/src/config/snapshot/access.rs @@ -94,6 +94,11 @@ impl<'repo> Snapshot<'repo> { /// Utilities impl<'repo> SnapshotMut<'repo> { + /// Apply all changes made to this instance. + /// + /// Note that this would also happen once this instance is dropped, but using this method may be more intuitive. + pub fn commit(self) {} + /// Don't apply any of the changes after consuming this instance, effectively forgetting them. pub fn forget(mut self) { std::mem::take(&mut self.config); diff --git a/git-repository/src/config/snapshot/apply_cli_overrides.rs b/git-repository/src/config/snapshot/apply_cli_overrides.rs index 7c775776e93..91aa1c54c8b 100644 --- a/git-repository/src/config/snapshot/apply_cli_overrides.rs +++ b/git-repository/src/config/snapshot/apply_cli_overrides.rs @@ -23,7 +23,10 @@ pub enum Error { impl SnapshotMut<'_> { /// Apply configuration values of the form `core.abbrev=5` or `remote.origin.url = foo` or `core.bool-implicit-true` /// to the repository configuration, marked with [source CLI][git_config::Source::Cli]. - pub fn apply_cli_overrides(&mut self, values: impl IntoIterator>) -> Result<(), Error> { + pub fn apply_cli_overrides( + &mut self, + values: impl IntoIterator>, + ) -> Result<&mut Self, Error> { let mut file = git_config::File::new(git_config::file::Metadata::from(git_config::Source::Cli)); for key_value in values { let key_value = key_value.as_ref(); @@ -44,6 +47,6 @@ impl SnapshotMut<'_> { ); } self.config.append(file); - Ok(()) + Ok(self) } } From 591afd56d9862a6348ef8b3af61798004b36aa19 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 23 Sep 2022 17:12:30 +0800 Subject: [PATCH 033/113] rename!: `bundle::write::Options::index_kind` -> `::index_version`. (#450) --- git-pack/src/bundle/write/mod.rs | 2 +- git-pack/src/bundle/write/types.rs | 4 ++-- git-pack/tests/pack/bundle.rs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/git-pack/src/bundle/write/mod.rs b/git-pack/src/bundle/write/mod.rs index 1abbd43f7fd..5d389fec9cc 100644 --- a/git-pack/src/bundle/write/mod.rs +++ b/git-pack/src/bundle/write/mod.rs @@ -228,7 +228,7 @@ impl crate::Bundle { Options { thread_limit, iteration_mode: _, - index_kind, + index_version: index_kind, object_hash, }: Options, data_file: Arc>>, diff --git a/git-pack/src/bundle/write/types.rs b/git-pack/src/bundle/write/types.rs index ade253302b7..bf4a22161bd 100644 --- a/git-pack/src/bundle/write/types.rs +++ b/git-pack/src/bundle/write/types.rs @@ -11,7 +11,7 @@ pub struct Options { /// Determine how much processing to spend on protecting against corruption or recovering from errors. pub iteration_mode: crate::data::input::Mode, /// The version of pack index to write, should be [`crate::index::Version::default()`] - pub index_kind: crate::index::Version, + pub index_version: crate::index::Version, /// The kind of hash to use when writing the bundle. pub object_hash: git_hash::Kind, } @@ -22,7 +22,7 @@ impl Default for Options { Options { thread_limit: None, iteration_mode: crate::data::input::Mode::Verify, - index_kind: Default::default(), + index_version: Default::default(), object_hash: Default::default(), } } diff --git a/git-pack/tests/pack/bundle.rs b/git-pack/tests/pack/bundle.rs index bf60aa844a9..04fa4bc136f 100644 --- a/git-pack/tests/pack/bundle.rs +++ b/git-pack/tests/pack/bundle.rs @@ -156,7 +156,7 @@ mod write_to_directory { pack::bundle::write::Options { thread_limit: None, iteration_mode: pack::data::input::Mode::Verify, - index_kind: pack::index::Version::V2, + index_version: pack::index::Version::V2, object_hash: git_hash::Kind::Sha1, }, ) From d5254c21083ae70c7a41229677dc85ccd28a2719 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 23 Sep 2022 17:13:17 +0800 Subject: [PATCH 034/113] adapt to changes in `git-pack` (#450) --- gitoxide-core/src/pack/index.rs | 2 +- gitoxide-core/src/pack/receive.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gitoxide-core/src/pack/index.rs b/gitoxide-core/src/pack/index.rs index a41e4e68746..b346039424e 100644 --- a/gitoxide-core/src/pack/index.rs +++ b/gitoxide-core/src/pack/index.rs @@ -85,7 +85,7 @@ pub fn from_pack( let options = pack::bundle::write::Options { thread_limit: ctx.thread_limit, iteration_mode: ctx.iteration_mode.into(), - index_kind: pack::index::Version::default(), + index_version: pack::index::Version::default(), object_hash: ctx.object_hash, }; let out = ctx.out; diff --git a/gitoxide-core/src/pack/receive.rs b/gitoxide-core/src/pack/receive.rs index 673196d48ee..e42634ddfac 100644 --- a/gitoxide-core/src/pack/receive.rs +++ b/gitoxide-core/src/pack/receive.rs @@ -340,7 +340,7 @@ fn receive_pack_blocking( ) -> io::Result<()> { let options = pack::bundle::write::Options { thread_limit: ctx.thread_limit, - index_kind: pack::index::Version::V2, + index_version: pack::index::Version::V2, iteration_mode: pack::data::input::Mode::Verify, object_hash: ctx.object_hash, }; From 5a3155a019ed5c9157cc699d4bbdf1b0b3623242 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 23 Sep 2022 17:16:49 +0800 Subject: [PATCH 035/113] obtain configuration for index version (with respect for lenient config) (#450) --- git-repository/src/open.rs | 2 +- git-repository/src/remote/connection/fetch.rs | 55 ++++++++++++++++++- 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/git-repository/src/open.rs b/git-repository/src/open.rs index 6ee8b0dd4ae..132ae8c8224 100644 --- a/git-repository/src/open.rs +++ b/git-repository/src/open.rs @@ -201,7 +201,7 @@ impl Options { /// If set, default is false, invalid configuration values will cause an error even if these can safely be defaulted. /// /// This is recommended for all applications that prefer correctness over usability. - /// `git` itself by defaults to strict configuration mode to let you know if configuration is incorrect. + /// `git` itself defaults to strict configuration mode, flagging incorrect configuration immediately. pub fn strict_config(mut self, toggle: bool) -> Self { self.lenient_config = !toggle; self diff --git a/git-repository/src/remote/connection/fetch.rs b/git-repository/src/remote/connection/fetch.rs index 43130ceef36..6e753a541c5 100644 --- a/git-repository/src/remote/connection/fetch.rs +++ b/git-repository/src/remote/connection/fetch.rs @@ -8,7 +8,13 @@ mod error { /// The error returned by [`receive()`](super::Prepare::receive()). #[derive(Debug, thiserror::Error)] #[error("TBD")] - pub enum Error {} + pub enum Error { + #[error("The configured pack.indexVersion is not valid. It must be 1 or 2, with 2 being the default{}", desired.map(|n| format!(" (but got {})", n)).unwrap_or_default())] + PackIndexVersion { + desired: Option, + source: Option, + }, + } } pub use error::Error; @@ -42,14 +48,59 @@ where T: Transport, P: Progress, { - /// receive the pack and perform the operation as configured by git via `git-config` or overridden by various builder methods. + /// Receive the pack and perform the operation as configured by git via `git-config` or overridden by various builder methods. + /// + /// ### Negotiation + /// + /// "fetch.negotiationAlgorithm" describes algorithms `git` uses currently, with the default being `consecutive` and `skipping` being + /// experimented with. We currently implement something we could call 'naive' which works for now. pub fn receive(mut self, _should_interrupt: &AtomicBool) -> Result<(), Error> { let mut con = self.con.take().expect("receive() can only be called once"); git_protocol::fetch::indicate_end_of_interaction(&mut con.transport).ok(); + + let repo = con.remote.repo; + let _index_version = config::pack_index_version(repo)?; + // let options = git_pack::bundle::write::Options { + // thread_limit: ctx.thread_limit, + // index_version: git_pack::index::Version::V2, + // iteration_mode: git_pack::data::input::Mode::Verify, + // object_hash: ctx.object_hash, + // }; + todo!() } } +mod config { + use super::Error; + use crate::Repository; + + pub fn pack_index_version(repo: &Repository) -> Result { + use git_pack::index::Version; + let lenient_config = repo.options.lenient_config; + Ok( + match repo.config.resolved.integer("pack", None, "indexVersion").transpose() { + Ok(Some(v)) if v == 1 => Version::V1, + Ok(Some(v)) if v == 2 => Version::V2, + Ok(None) => Version::V2, + Ok(Some(_)) | Err(_) if lenient_config => Version::V2, + Ok(Some(v)) => { + return Err(Error::PackIndexVersion { + desired: v.into(), + source: None, + }) + } + Err(err) => { + return Err(Error::PackIndexVersion { + desired: None, + source: err.into(), + }) + } + }, + ) + } +} + /// A structure to hold the result of the handshake with the remote and configure the upcoming fetch operation. #[allow(dead_code)] pub struct Prepare<'remote, 'repo, T, P> From 8d17dc68cea8a6b6b417f12d45fcf4331cf562fd Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 23 Sep 2022 17:30:07 +0800 Subject: [PATCH 036/113] also extract index threads (#450) --- .../src/remote/connection/fetch/config.rs | 66 +++++++++++++++++++ .../connection/{fetch.rs => fetch/mod.rs} | 36 ++-------- 2 files changed, 71 insertions(+), 31 deletions(-) create mode 100644 git-repository/src/remote/connection/fetch/config.rs rename git-repository/src/remote/connection/{fetch.rs => fetch/mod.rs} (73%) diff --git a/git-repository/src/remote/connection/fetch/config.rs b/git-repository/src/remote/connection/fetch/config.rs new file mode 100644 index 00000000000..dadb5da2dd6 --- /dev/null +++ b/git-repository/src/remote/connection/fetch/config.rs @@ -0,0 +1,66 @@ +use super::Error; +use crate::Repository; +use std::convert::TryInto; + +pub fn index_threads(repo: &Repository) -> Result, Error> { + let lenient_config = repo.options.lenient_config; + let message = "The configured pack.threads is invalid. It must be 0 or greater, with 0 turning it off"; + Ok( + match repo + .config + .resolved + .integer_filter("pack", None, "threads", &mut repo.filter_config_section()) + .transpose() + { + Ok(Some(0)) => Some(1), + Ok(Some(n)) => match n.try_into() { + Ok(n) => Some(n), + Err(_) if lenient_config => None, + Err(_) => { + return Err(Error::Configuration { + message: "The value for pack.threads is out of range", + desired: n.into(), + source: None, + }) + } + }, + Ok(None) => None, + Err(_) if lenient_config => None, + Err(err) => { + return Err(Error::Configuration { + message, + desired: None, + source: err.into(), + }) + } + }, + ) +} + +pub fn pack_index_version(repo: &Repository) -> Result { + use git_pack::index::Version; + let lenient_config = repo.options.lenient_config; + let message = "The configured pack.indexVersion is invalid. It must be 1 or 2, with 2 being the default"; + Ok( + match repo.config.resolved.integer("pack", None, "indexVersion").transpose() { + Ok(Some(v)) if v == 1 => Version::V1, + Ok(Some(v)) if v == 2 => Version::V2, + Ok(None) => Version::V2, + Ok(Some(_)) | Err(_) if lenient_config => Version::V2, + Ok(Some(v)) => { + return Err(Error::Configuration { + message, + desired: v.into(), + source: None, + }) + } + Err(err) => { + return Err(Error::Configuration { + message, + desired: None, + source: err.into(), + }) + } + }, + ) +} diff --git a/git-repository/src/remote/connection/fetch.rs b/git-repository/src/remote/connection/fetch/mod.rs similarity index 73% rename from git-repository/src/remote/connection/fetch.rs rename to git-repository/src/remote/connection/fetch/mod.rs index 6e753a541c5..29fdb0c81ce 100644 --- a/git-repository/src/remote/connection/fetch.rs +++ b/git-repository/src/remote/connection/fetch/mod.rs @@ -9,8 +9,9 @@ mod error { #[derive(Debug, thiserror::Error)] #[error("TBD")] pub enum Error { - #[error("The configured pack.indexVersion is not valid. It must be 1 or 2, with 2 being the default{}", desired.map(|n| format!(" (but got {})", n)).unwrap_or_default())] - PackIndexVersion { + #[error("{message}{}", desired.map(|n| format!(" (got {})", n)).unwrap_or_default())] + Configuration { + message: &'static str, desired: Option, source: Option, }, @@ -60,6 +61,7 @@ where let repo = con.remote.repo; let _index_version = config::pack_index_version(repo)?; + let _thread_limit = config::index_threads(repo)?; // let options = git_pack::bundle::write::Options { // thread_limit: ctx.thread_limit, // index_version: git_pack::index::Version::V2, @@ -71,35 +73,7 @@ where } } -mod config { - use super::Error; - use crate::Repository; - - pub fn pack_index_version(repo: &Repository) -> Result { - use git_pack::index::Version; - let lenient_config = repo.options.lenient_config; - Ok( - match repo.config.resolved.integer("pack", None, "indexVersion").transpose() { - Ok(Some(v)) if v == 1 => Version::V1, - Ok(Some(v)) if v == 2 => Version::V2, - Ok(None) => Version::V2, - Ok(Some(_)) | Err(_) if lenient_config => Version::V2, - Ok(Some(v)) => { - return Err(Error::PackIndexVersion { - desired: v.into(), - source: None, - }) - } - Err(err) => { - return Err(Error::PackIndexVersion { - desired: None, - source: err.into(), - }) - } - }, - ) - } -} +mod config; /// A structure to hold the result of the handshake with the remote and configure the upcoming fetch operation. #[allow(dead_code)] From 92e082a288cf5e48caa205a4c7bd1ced025fea46 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 23 Sep 2022 19:27:11 +0800 Subject: [PATCH 037/113] A very first version of `gix progress show` (#450) --- Cargo.lock | 7 + Cargo.toml | 3 + README.md | 4 + src/plumbing/main.rs | 9 +- src/plumbing/mod.rs | 4 + src/plumbing/options.rs | 677 ----------------------------------- src/plumbing/options/free.rs | 368 +++++++++++++++++++ src/plumbing/options/mod.rs | 319 +++++++++++++++++ src/plumbing/progress.rs | 44 +++ 9 files changed, 756 insertions(+), 679 deletions(-) delete mode 100644 src/plumbing/options.rs create mode 100644 src/plumbing/options/free.rs create mode 100644 src/plumbing/options/mod.rs create mode 100644 src/plumbing/progress.rs diff --git a/Cargo.lock b/Cargo.lock index a01d5d418a4..73196abb6c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1835,6 +1835,7 @@ dependencies = [ "git-repository", "git-transport", "gitoxide-core", + "owo-colors", "prodash", ] @@ -2380,6 +2381,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "owo-colors" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" + [[package]] name = "parking" version = "2.0.0" diff --git a/Cargo.toml b/Cargo.toml index 80013bf8cfa..96ce65e2e7e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -92,6 +92,9 @@ env_logger = { version = "0.9.0", default-features = false } crosstermion = { version = "0.10.1", optional = true, default-features = false } futures-lite = { version = "1.12.0", optional = true, default-features = false, features = ["std"] } +# for show-progress +owo-colors = "3.5.0" + document-features = { version = "0.2.0", optional = true } [profile.dev.package] diff --git a/README.md b/README.md index da00f4e2143..37c2218f0db 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,10 @@ Please see _'Development Status'_ for a listing of all crates and their capabili * Based on the [git-hours] algorithm. * See the [discussion][git-hours-discussion] for some performance data. * **the `gix` program** _(plumbing)_ - lower level commands for use in automation + * **progress** + * [x] **show** - provide an overview of what works and what doesn't from the perspective of the git configuration. + This is likely to change a lot over time depending on actual needs, but maybe useful for you to see + if particular git-configuration is picked up and where it deviates. * **config** - list the complete git configuration in human-readable form and optionally filter sections by name. * **exclude** * [x] **query** - check if path specs are excluded via gits exclusion rules like `.gitignore`. diff --git a/src/plumbing/main.rs b/src/plumbing/main.rs index 5c587d488dd..efc8ed7ea85 100644 --- a/src/plumbing/main.rs +++ b/src/plumbing/main.rs @@ -14,8 +14,12 @@ use gitoxide_core as core; use gitoxide_core::pack::verify; use crate::{ - plumbing::options::{ - commit, config, credential, exclude, free, mailmap, odb, remote, revision, tree, Args, Subcommands, + plumbing::{ + options::{ + commit, config, credential, exclude, free, mailmap, odb, progress, remote, revision, tree, Args, + Subcommands, + }, + show_progress, }, shared::pretty::prepare_and_run, }; @@ -111,6 +115,7 @@ pub fn main() -> Result<()> { })?; match cmd { + Subcommands::Progress(progress::Subcommands::Show) => show_progress(), Subcommands::Credential(cmd) => core::repository::credential( repository(Mode::StrictWithGitInstallConfig)?, match cmd { diff --git a/src/plumbing/mod.rs b/src/plumbing/mod.rs index 2e32346bdd9..d2fab42bb0e 100644 --- a/src/plumbing/mod.rs +++ b/src/plumbing/mod.rs @@ -1,4 +1,8 @@ mod main; pub use main::main; +#[path = "progress.rs"] +mod progress_impl; +pub use progress_impl::show_progress; + mod options; diff --git a/src/plumbing/options.rs b/src/plumbing/options.rs deleted file mode 100644 index 663773cc745..00000000000 --- a/src/plumbing/options.rs +++ /dev/null @@ -1,677 +0,0 @@ -use std::path::PathBuf; - -use git_repository as git; -use git_repository::bstr::BString; -use gitoxide_core as core; - -#[derive(Debug, clap::Parser)] -#[clap(name = "gix-plumbing", about = "The git underworld", version = clap::crate_version!())] -#[clap(subcommand_required = true)] -#[clap(arg_required_else_help = true)] -pub struct Args { - /// The repository to access. - #[clap(short = 'r', long, default_value = ".")] - pub repository: PathBuf, - - /// Add these values to the configuration in the form of `key=value` or `key`. - /// - /// For example, if `key` is `core.abbrev`, set configuration like `[core] abbrev = key`, - /// or `remote.origin.url = foo` to set `[remote "origin"] url = foo`. - #[clap(long, short = 'c', parse(try_from_os_str = git::env::os_str_to_bstring))] - pub config: Vec, - - #[clap(long, short = 't')] - /// The amount of threads to use for some operations. - /// - /// If unset, or the value is 0, there is no limit and all logical cores can be used. - pub threads: Option, - - /// Display verbose messages and progress information - #[clap(long, short = 'v')] - pub verbose: bool, - - /// Bring up a terminal user interface displaying progress visually - #[cfg(feature = "prodash-render-tui")] - #[clap(long, conflicts_with("verbose"))] - pub progress: bool, - - /// The progress TUI will stay up even though the work is already completed. - /// - /// Use this to be able to read progress messages or additional information visible in the TUI log pane. - #[cfg(feature = "prodash-render-tui")] - #[clap(long, conflicts_with("verbose"), requires("progress"))] - pub progress_keep_open: bool, - - /// Determine the format to use when outputting statistics. - #[clap( - long, - short = 'f', - default_value = "human", - possible_values(core::OutputFormat::variants()) - )] - pub format: core::OutputFormat, - - /// The object format to assume when reading files that don't inherently know about it, or when writing files. - #[clap(long, default_value_t = git_repository::hash::Kind::default(), possible_values(&["SHA1"]))] - pub object_hash: git_repository::hash::Kind, - - #[clap(subcommand)] - pub cmd: Subcommands, -} - -#[derive(Debug, clap::Subcommand)] -pub enum Subcommands { - /// Interact with the object database. - #[clap(subcommand)] - Odb(odb::Subcommands), - /// Interact with tree objects. - #[clap(subcommand)] - Tree(tree::Subcommands), - /// Interact with commit objects. - #[clap(subcommand)] - Commit(commit::Subcommands), - /// Verify the integrity of the entire repository - Verify { - #[clap(flatten)] - args: free::pack::VerifyOptions, - }, - /// Query and obtain information about revisions. - #[clap(subcommand)] - Revision(revision::Subcommands), - /// A program just like `git credential`. - #[clap(subcommand)] - Credential(credential::Subcommands), - /// Interact with the mailmap. - #[clap(subcommand)] - Mailmap(mailmap::Subcommands), - /// Interact with the remote hosts. - Remote(remote::Platform), - /// Interact with the exclude files like .gitignore. - #[clap(subcommand)] - Exclude(exclude::Subcommands), - Config(config::Platform), - /// Subcommands that need no git repository to run. - #[clap(subcommand)] - Free(free::Subcommands), -} - -pub mod config { - /// Print all entries in a configuration file or access other sub-commands - #[derive(Debug, clap::Parser)] - #[clap(subcommand_required(false))] - pub struct Platform { - /// The filter terms to limit the output to matching sections and subsections only. - /// - /// Typical filters are `branch` or `remote.origin` or `remote.or*` - git-style globs are supported - /// and comparisons are case-insensitive. - pub filter: Vec, - } -} - -pub mod remote { - use git_repository as git; - - #[derive(Debug, clap::Parser)] - pub struct Platform { - /// The name of the remote to connect to. - /// - /// If unset, the current branch will determine the remote. - #[clap(long, short = 'n')] - pub name: Option, - - /// Connect directly to the given URL, forgoing any configuration from the repository. - #[clap(long, short = 'u', conflicts_with("name"), parse(try_from_os_str = std::convert::TryFrom::try_from))] - pub url: Option, - - /// Output additional typically information provided by the server as part of the connection handshake. - #[clap(long)] - pub handshake_info: bool, - - /// Subcommands - #[clap(subcommand)] - pub cmd: Subcommands, - } - - #[derive(Debug, clap::Subcommand)] - #[clap(visible_alias = "remotes")] - pub enum Subcommands { - /// Print all references available on the remote. - #[cfg(any(feature = "gitoxide-core-async-client", feature = "gitoxide-core-blocking-client"))] - Refs, - /// Print all references available on the remote as filtered through ref-specs. - #[cfg(any(feature = "gitoxide-core-async-client", feature = "gitoxide-core-blocking-client"))] - RefMap { - /// Override the built-in and configured ref-specs with one or more of the given ones. - #[clap(parse(try_from_os_str = git::env::os_str_to_bstring))] - ref_spec: Vec, - }, - } -} - -pub mod mailmap { - #[derive(Debug, clap::Subcommand)] - pub enum Subcommands { - /// Print all entries in configured mailmaps, inform about errors as well. - Entries, - } -} - -pub mod odb { - #[derive(Debug, clap::Subcommand)] - pub enum Subcommands { - /// Print all object names. - Entries, - /// Provide general information about the object database. - Info, - } -} - -pub mod tree { - #[derive(Debug, clap::Subcommand)] - pub enum Subcommands { - /// Print entries in a given tree - Entries { - /// Traverse the entire tree and its subtrees respectively, not only this tree. - #[clap(long, short = 'r')] - recursive: bool, - - /// Provide files size as well. This is expensive as the object is decoded entirely. - #[clap(long, short = 'e')] - extended: bool, - - /// The tree to traverse, or the tree at `HEAD` if unspecified. - treeish: Option, - }, - /// Provide information about a tree. - Info { - /// Provide files size as well. This is expensive as the object is decoded entirely. - #[clap(long, short = 'e')] - extended: bool, - /// The tree to traverse, or the tree at `HEAD` if unspecified. - treeish: Option, - }, - } -} - -pub mod commit { - #[derive(Debug, clap::Subcommand)] - pub enum Subcommands { - /// Describe the current commit or the given one using the name of the closest annotated tag in its ancestry. - Describe { - /// Use annotated tag references only, not all tags. - #[clap(long, short = 't', conflicts_with("all-refs"))] - annotated_tags: bool, - - /// Use all references under the `ref/` namespaces, which includes tag references, local and remote branches. - #[clap(long, short = 'a', conflicts_with("annotated-tags"))] - all_refs: bool, - - /// Only follow the first parent when traversing the commit graph. - #[clap(long, short = 'f')] - first_parent: bool, - - /// Always display the long format, even if that would not be necessary as the id is located directly on a reference. - #[clap(long, short = 'l')] - long: bool, - - /// Consider only the given `n` candidates. This can take longer, but potentially produces more accurate results. - #[clap(long, short = 'c', default_value = "10")] - max_candidates: usize, - - /// Print information on stderr to inform about performance statistics - #[clap(long, short = 's')] - statistics: bool, - - #[clap(long)] - /// If there was no way to describe the commit, fallback to using the abbreviated input revision. - always: bool, - - /// A specification of the revision to use, or the current `HEAD` if unset. - rev_spec: Option, - }, - } -} - -pub mod credential { - #[derive(Debug, clap::Subcommand)] - pub enum Subcommands { - /// Get the credentials fed for `url=` via STDIN. - #[clap(visible_alias = "get")] - Fill, - /// Approve the information piped via STDIN as obtained with last call to `fill` - #[clap(visible_alias = "store")] - Approve, - /// Try to resolve the given revspec and print the object names. - #[clap(visible_alias = "erase")] - Reject, - } -} - -pub mod revision { - #[derive(Debug, clap::Subcommand)] - #[clap(visible_alias = "rev", visible_alias = "r")] - pub enum Subcommands { - /// List all commits reachable from the given rev-spec. - #[clap(visible_alias = "l")] - List { spec: std::ffi::OsString }, - /// Provide the revision specification like `@~1` to explain. - #[clap(visible_alias = "e")] - Explain { spec: std::ffi::OsString }, - /// Try to resolve the given revspec and print the object names. - #[clap(visible_alias = "query", visible_alias = "parse", visible_alias = "p")] - Resolve { - /// Instead of resolving a rev-spec, explain what would be done for the first spec. - /// - /// Equivalent to the `explain` subcommand. - #[clap(short = 'e', long)] - explain: bool, - /// Show the first resulting object similar to how `git cat-file` would, but don't show the resolved spec. - #[clap(short = 'c', long, conflicts_with = "explain")] - cat_file: bool, - /// rev-specs like `@`, `@~1` or `HEAD^2`. - #[clap(required = true)] - specs: Vec, - }, - /// Return the names and hashes of all previously checked-out branches. - #[clap(visible_alias = "prev")] - PreviousBranches, - } -} - -/// -pub mod free { - #[derive(Debug, clap::Subcommand)] - #[clap(visible_alias = "no-repo")] - pub enum Subcommands { - /// Subcommands for interacting with commit-graphs - #[clap(subcommand)] - CommitGraph(commitgraph::Subcommands), - /// Subcommands for interacting with mailmaps - Mailmap { - #[clap(flatten)] - cmd: mailmap::Platform, - }, - /// Subcommands for interacting with pack files and indices - #[clap(subcommand)] - Pack(pack::Subcommands), - /// Subcommands for interacting with a worktree index, typically at .git/index - Index(index::Platform), - } - - /// - pub mod commitgraph { - use std::path::PathBuf; - - #[derive(Debug, clap::Subcommand)] - pub enum Subcommands { - /// Verify the integrity of a commit graph - Verify { - /// The path to '.git/objects/info/', '.git/objects/info/commit-graphs/', or '.git/objects/info/commit-graph' to validate. - path: PathBuf, - /// output statistical information about the pack - #[clap(long, short = 's')] - statistics: bool, - }, - } - } - - pub mod index { - use std::path::PathBuf; - - #[derive(Debug, clap::Parser)] - pub struct Platform { - /// The object format to assume when reading files that don't inherently know about it, or when writing files. - #[clap(long, default_value_t = git_repository::hash::Kind::default(), possible_values(&["SHA1"]))] - pub object_hash: git_repository::hash::Kind, - - /// The path to the index file. - #[clap(short = 'i', long, default_value = ".git/index")] - pub index_path: PathBuf, - - /// Subcommands - #[clap(subcommand)] - pub cmd: Subcommands, - } - - #[derive(Debug, clap::Subcommand)] - pub enum Subcommands { - /// Validate constraints and assumptions of an index along with its integrity. - Verify, - /// Print all entries to standard output - Entries, - /// Print information about the index structure - Info { - /// Do not extract specific extension information to gain only a superficial idea of the index's composition. - #[clap(long)] - no_details: bool, - }, - /// Checkout the index into a directory with exclusive write access, similar to what would happen during clone. - CheckoutExclusive { - /// The path to `.git` repository from which objects can be obtained to write the actual files referenced - /// in the index. Use this measure the impact on extracting objects on overall performance. - #[clap(long, short = 'r')] - repository: Option, - /// Ignore errors and keep checking out as many files as possible, and report all errors at the end of the operation. - #[clap(long, short = 'k')] - keep_going: bool, - /// Enable to query the object database yet write only empty files. This is useful to measure the overhead of ODB query - /// compared to writing the bytes to disk. - #[clap(long, short = 'e', requires = "repository")] - empty_files: bool, - /// The directory into which to write all index entries. - directory: PathBuf, - }, - } - } - - /// - pub mod pack { - use std::{ffi::OsString, path::PathBuf}; - - use gitoxide_core as core; - - #[derive(Debug, clap::Subcommand)] - pub enum Subcommands { - /// Subcommands for interacting with pack indices (.idx) - #[clap(subcommand)] - Index(index::Subcommands), - /// Subcommands for interacting with multi-pack indices (named "multi-pack-index") - MultiIndex(multi_index::Platform), - /// Create a new pack with a set of objects. - Create { - #[clap(long, short = 'r')] - /// the directory containing the '.git' repository from which objects should be read. - repository: Option, - - #[clap(long, short = 'e', possible_values(core::pack::create::ObjectExpansion::variants()))] - /// the way objects are expanded. They differ in costs. - /// - /// Possible values are "none" and "tree-traversal". Default is "none". - expansion: Option, - - #[clap(long, default_value_t = 3, requires = "nondeterministic-count")] - /// The amount of threads to use when counting and the `--nondeterminisitc-count` flag is set, defaulting - /// to the globally configured threads. - /// - /// Use it to have different trade-offs between counting performance and cost in terms of CPU, as the scaling - /// here is everything but linear. The effectiveness of each core seems to be no more than 30%. - counting_threads: usize, - - #[clap(long)] - /// if set, the counting phase may be accelerated using multithreading. - /// - /// On the flip side, however, one will loose deterministic counting results which affects the - /// way the resulting pack is structured. - nondeterministic_count: bool, - - #[clap(long, short = 's')] - /// If set statistical information will be presented to inform about pack creation details. - /// It's a form of instrumentation for developers to help improve pack generation. - statistics: bool, - - #[clap(long)] - /// The size in megabytes for a cache to speed up pack access for packs with long delta chains. - /// It is shared among all threads, so 4 threads would use their own cache 1/4th of the size. - /// - /// If unset, no cache will be used. - pack_cache_size_mb: Option, - - #[clap(long)] - /// The size in megabytes for a cache to speed up accessing entire objects, bypassing object database access when hit. - /// It is shared among all threads, so 4 threads would use their own cache 1/4th of the size. - /// - /// This cache type is currently only effective when using the 'diff-tree' object expansion. - /// - /// If unset, no cache will be used. - object_cache_size_mb: Option, - - #[clap(long)] - /// if set, delta-objects whose base object wouldn't be in the pack will not be recompressed as base object, but instead - /// refer to its base object using its object id. - /// - /// This allows for smaller packs but requires the receiver of the pack to resolve these ids before storing the pack. - /// Packs produced with this option enabled are only valid in transit, but not at rest. - thin: bool, - - /// The directory into which to write the pack file. - #[clap(long, short = 'o')] - output_directory: Option, - - /// The tips from which to start the commit graph iteration, either as fully qualified commit hashes - /// or as branch names. - /// - /// If empty, we expect to read objects on stdin and default to 'none' as expansion mode. - /// Otherwise the expansion mode is 'tree-traversal' by default. - tips: Vec, - }, - /// Use the git-protocol to receive a pack, emulating a clone. - #[cfg(any(feature = "gitoxide-core-async-client", feature = "gitoxide-core-blocking-client"))] - Receive { - /// The protocol version to use. Valid values are 1 and 2 - #[clap(long, short = 'p')] - protocol: Option, - - /// the directory into which to write references. Existing files will be overwritten. - /// - /// Note that the directory will be created if needed. - #[clap(long, short = 'd')] - refs_directory: Option, - - /// The URLs or path from which to receive the pack. - /// - /// See here for a list of supported URLs: - url: String, - - /// If set once or more times, these references will be fetched instead of all advertised ones. - /// - /// Note that this requires a reasonably modern git server. - #[clap(long = "reference", short = 'r')] - refs: Vec, - - /// The directory into which to write the received pack and index. - /// - /// If unset, they will be discarded. - directory: Option, - }, - /// Dissolve a pack into its loose objects. - /// - /// Note that this effectively removes delta compression for an average compression of 2x, creating one file per object in the process. - /// Thus this should only be done to dissolve small packs after a fetch. - Explode { - #[clap(long)] - /// Read written objects back and assert they match their source. Fail the operation otherwise. - /// - /// Only relevant if an object directory is set. - verify: bool, - - /// delete the pack and index file after the operation is successful - #[clap(long)] - delete_pack: bool, - - /// The amount of checks to run - #[clap( - long, - short = 'c', - default_value = "all", - possible_values(core::pack::explode::SafetyCheck::variants()) - )] - check: core::pack::explode::SafetyCheck, - - /// Compress bytes even when using the sink, i.e. no object directory is specified - /// - /// This helps to determine overhead related to compression. If unset, the sink will - /// only create hashes from bytes, which is usually limited by the speed at which input - /// can be obtained. - #[clap(long)] - sink_compress: bool, - - /// The '.pack' or '.idx' file to explode into loose objects - pack_path: PathBuf, - - /// The path into which all objects should be written. Commonly '.git/objects' - object_path: Option, - }, - /// Verify the integrity of a pack, index or multi-index file - Verify { - #[clap(flatten)] - args: VerifyOptions, - - /// The '.pack', '.idx' or 'multi-pack-index' file to validate. - path: PathBuf, - }, - } - - #[derive(Debug, clap::Parser)] - pub struct VerifyOptions { - /// output statistical information - #[clap(long, short = 's')] - pub statistics: bool, - /// The algorithm used to verify packs. They differ in costs. - #[clap( - long, - short = 'a', - default_value = "less-time", - possible_values(core::pack::verify::Algorithm::variants()) - )] - pub algorithm: core::pack::verify::Algorithm, - - #[clap(long, conflicts_with("re-encode"))] - /// Decode and parse tags, commits and trees to validate their correctness beyond hashing correctly. - /// - /// Malformed objects should not usually occur, but could be injected on purpose or accident. - /// This will reduce overall performance. - pub decode: bool, - - #[clap(long)] - /// Decode and parse tags, commits and trees to validate their correctness, and re-encode them. - /// - /// This flag is primarily to test the implementation of encoding, and requires to decode the object first. - /// Encoding an object after decoding it should yield exactly the same bytes. - /// This will reduce overall performance even more, as re-encoding requires to transform zero-copy objects into - /// owned objects, causing plenty of allocation to occur. - pub re_encode: bool, - } - - /// - pub mod multi_index { - use std::path::PathBuf; - - #[derive(Debug, clap::Parser)] - pub struct Platform { - /// The path to the index file. - #[clap(short = 'i', long, default_value = ".git/objects/pack/multi-pack-index")] - pub multi_index_path: PathBuf, - - /// Subcommands - #[clap(subcommand)] - pub cmd: Subcommands, - } - - #[derive(Debug, clap::Subcommand)] - pub enum Subcommands { - /// Display all entries of a multi-index: - Entries, - /// Print general information about a multi-index file - Info, - /// Verify a multi-index quickly without inspecting objects themselves - Verify, - /// Create a multi-pack index from one or more pack index files, overwriting possibloy existing files. - Create { - /// Paths to the pack index files to read (with .idx extension). - /// - /// Note for the multi-index to be useful, it should be side-by-side with the supplied `.idx` files. - #[clap(required = true)] - index_paths: Vec, - }, - } - } - - /// - pub mod index { - use std::path::PathBuf; - - use gitoxide_core as core; - - #[derive(Debug, clap::Subcommand)] - pub enum Subcommands { - /// create a pack index from a pack data file. - Create { - /// Specify how to iterate the pack, defaults to 'verify' - /// - /// Valid values are - /// - /// **as-is** do not do anything and expect the pack file to be valid as per the trailing hash, - /// **verify** the input ourselves and validate that it matches with the hash provided in the pack, - /// **restore** hash the input ourselves and ignore failing entries, instead finish the pack with the hash we computed - /// to keep as many objects as possible. - #[clap( - long, - short = 'i', - default_value = "verify", - possible_values(core::pack::index::IterationMode::variants()) - )] - iteration_mode: core::pack::index::IterationMode, - - /// Path to the pack file to read (with .pack extension). - /// - /// If unset, the pack file is expected on stdin. - #[clap(long, short = 'p')] - pack_path: Option, - - /// The folder into which to place the pack and the generated index file - /// - /// If unset, only informational output will be provided to standard output. - directory: Option, - }, - } - } - } - - /// - pub mod mailmap { - use std::path::PathBuf; - - #[derive(Debug, clap::Parser)] - pub struct Platform { - /// The path to the mailmap file. - #[clap(short = 'p', long, default_value = ".mailmap")] - pub path: PathBuf, - - /// Subcommands - #[clap(subcommand)] - pub cmd: Subcommands, - } - - #[derive(Debug, clap::Subcommand)] - pub enum Subcommands { - /// Parse all entries in the mailmap and report malformed lines or collisions. - Verify, - } - } -} - -pub mod exclude { - use std::ffi::OsString; - - use git_repository as git; - - #[derive(Debug, clap::Subcommand)] - pub enum Subcommands { - /// Check if path-specs are excluded and print the result similar to `git check-ignore`. - Query { - /// Show actual ignore patterns instead of un-excluding an entry. - /// - /// That way one can understand why an entry might not be excluded. - #[clap(long, short = 'i')] - show_ignore_patterns: bool, - /// Additional patterns to use for exclusions. They have the highest priority. - /// - /// Useful for undoing previous patterns using the '!' prefix. - #[clap(long, short = 'p')] - patterns: Vec, - /// The git path specifications to check for exclusion, or unset to read from stdin one per line. - #[clap(parse(try_from_os_str = std::convert::TryFrom::try_from))] - pathspecs: Vec, - }, - } -} diff --git a/src/plumbing/options/free.rs b/src/plumbing/options/free.rs new file mode 100644 index 00000000000..7e8e25c83ed --- /dev/null +++ b/src/plumbing/options/free.rs @@ -0,0 +1,368 @@ +#[derive(Debug, clap::Subcommand)] +#[clap(visible_alias = "no-repo")] +pub enum Subcommands { + /// Subcommands for interacting with commit-graphs + #[clap(subcommand)] + CommitGraph(commitgraph::Subcommands), + /// Subcommands for interacting with mailmaps + Mailmap { + #[clap(flatten)] + cmd: mailmap::Platform, + }, + /// Subcommands for interacting with pack files and indices + #[clap(subcommand)] + Pack(pack::Subcommands), + /// Subcommands for interacting with a worktree index, typically at .git/index + Index(index::Platform), +} + +/// +pub mod commitgraph { + use std::path::PathBuf; + + #[derive(Debug, clap::Subcommand)] + pub enum Subcommands { + /// Verify the integrity of a commit graph + Verify { + /// The path to '.git/objects/info/', '.git/objects/info/commit-graphs/', or '.git/objects/info/commit-graph' to validate. + path: PathBuf, + /// output statistical information about the pack + #[clap(long, short = 's')] + statistics: bool, + }, + } +} + +pub mod index { + use std::path::PathBuf; + + #[derive(Debug, clap::Parser)] + pub struct Platform { + /// The object format to assume when reading files that don't inherently know about it, or when writing files. + #[clap(long, default_value_t = git_repository::hash::Kind::default(), possible_values(&["SHA1"]))] + pub object_hash: git_repository::hash::Kind, + + /// The path to the index file. + #[clap(short = 'i', long, default_value = ".git/index")] + pub index_path: PathBuf, + + /// Subcommands + #[clap(subcommand)] + pub cmd: Subcommands, + } + + #[derive(Debug, clap::Subcommand)] + pub enum Subcommands { + /// Validate constraints and assumptions of an index along with its integrity. + Verify, + /// Print all entries to standard output + Entries, + /// Print information about the index structure + Info { + /// Do not extract specific extension information to gain only a superficial idea of the index's composition. + #[clap(long)] + no_details: bool, + }, + /// Checkout the index into a directory with exclusive write access, similar to what would happen during clone. + CheckoutExclusive { + /// The path to `.git` repository from which objects can be obtained to write the actual files referenced + /// in the index. Use this measure the impact on extracting objects on overall performance. + #[clap(long, short = 'r')] + repository: Option, + /// Ignore errors and keep checking out as many files as possible, and report all errors at the end of the operation. + #[clap(long, short = 'k')] + keep_going: bool, + /// Enable to query the object database yet write only empty files. This is useful to measure the overhead of ODB query + /// compared to writing the bytes to disk. + #[clap(long, short = 'e', requires = "repository")] + empty_files: bool, + /// The directory into which to write all index entries. + directory: PathBuf, + }, + } +} + +/// +pub mod pack { + use std::{ffi::OsString, path::PathBuf}; + + use gitoxide_core as core; + + #[derive(Debug, clap::Subcommand)] + pub enum Subcommands { + /// Subcommands for interacting with pack indices (.idx) + #[clap(subcommand)] + Index(index::Subcommands), + /// Subcommands for interacting with multi-pack indices (named "multi-pack-index") + MultiIndex(multi_index::Platform), + /// Create a new pack with a set of objects. + Create { + #[clap(long, short = 'r')] + /// the directory containing the '.git' repository from which objects should be read. + repository: Option, + + #[clap(long, short = 'e', possible_values(core::pack::create::ObjectExpansion::variants()))] + /// the way objects are expanded. They differ in costs. + /// + /// Possible values are "none" and "tree-traversal". Default is "none". + expansion: Option, + + #[clap(long, default_value_t = 3, requires = "nondeterministic-count")] + /// The amount of threads to use when counting and the `--nondeterminisitc-count` flag is set, defaulting + /// to the globally configured threads. + /// + /// Use it to have different trade-offs between counting performance and cost in terms of CPU, as the scaling + /// here is everything but linear. The effectiveness of each core seems to be no more than 30%. + counting_threads: usize, + + #[clap(long)] + /// if set, the counting phase may be accelerated using multithreading. + /// + /// On the flip side, however, one will loose deterministic counting results which affects the + /// way the resulting pack is structured. + nondeterministic_count: bool, + + #[clap(long, short = 's')] + /// If set statistical information will be presented to inform about pack creation details. + /// It's a form of instrumentation for developers to help improve pack generation. + statistics: bool, + + #[clap(long)] + /// The size in megabytes for a cache to speed up pack access for packs with long delta chains. + /// It is shared among all threads, so 4 threads would use their own cache 1/4th of the size. + /// + /// If unset, no cache will be used. + pack_cache_size_mb: Option, + + #[clap(long)] + /// The size in megabytes for a cache to speed up accessing entire objects, bypassing object database access when hit. + /// It is shared among all threads, so 4 threads would use their own cache 1/4th of the size. + /// + /// This cache type is currently only effective when using the 'diff-tree' object expansion. + /// + /// If unset, no cache will be used. + object_cache_size_mb: Option, + + #[clap(long)] + /// if set, delta-objects whose base object wouldn't be in the pack will not be recompressed as base object, but instead + /// refer to its base object using its object id. + /// + /// This allows for smaller packs but requires the receiver of the pack to resolve these ids before storing the pack. + /// Packs produced with this option enabled are only valid in transit, but not at rest. + thin: bool, + + /// The directory into which to write the pack file. + #[clap(long, short = 'o')] + output_directory: Option, + + /// The tips from which to start the commit graph iteration, either as fully qualified commit hashes + /// or as branch names. + /// + /// If empty, we expect to read objects on stdin and default to 'none' as expansion mode. + /// Otherwise the expansion mode is 'tree-traversal' by default. + tips: Vec, + }, + /// Use the git-protocol to receive a pack, emulating a clone. + #[cfg(any(feature = "gitoxide-core-async-client", feature = "gitoxide-core-blocking-client"))] + Receive { + /// The protocol version to use. Valid values are 1 and 2 + #[clap(long, short = 'p')] + protocol: Option, + + /// the directory into which to write references. Existing files will be overwritten. + /// + /// Note that the directory will be created if needed. + #[clap(long, short = 'd')] + refs_directory: Option, + + /// The URLs or path from which to receive the pack. + /// + /// See here for a list of supported URLs: + url: String, + + /// If set once or more times, these references will be fetched instead of all advertised ones. + /// + /// Note that this requires a reasonably modern git server. + #[clap(long = "reference", short = 'r')] + refs: Vec, + + /// The directory into which to write the received pack and index. + /// + /// If unset, they will be discarded. + directory: Option, + }, + /// Dissolve a pack into its loose objects. + /// + /// Note that this effectively removes delta compression for an average compression of 2x, creating one file per object in the process. + /// Thus this should only be done to dissolve small packs after a fetch. + Explode { + #[clap(long)] + /// Read written objects back and assert they match their source. Fail the operation otherwise. + /// + /// Only relevant if an object directory is set. + verify: bool, + + /// delete the pack and index file after the operation is successful + #[clap(long)] + delete_pack: bool, + + /// The amount of checks to run + #[clap( + long, + short = 'c', + default_value = "all", + possible_values(core::pack::explode::SafetyCheck::variants()) + )] + check: core::pack::explode::SafetyCheck, + + /// Compress bytes even when using the sink, i.e. no object directory is specified + /// + /// This helps to determine overhead related to compression. If unset, the sink will + /// only create hashes from bytes, which is usually limited by the speed at which input + /// can be obtained. + #[clap(long)] + sink_compress: bool, + + /// The '.pack' or '.idx' file to explode into loose objects + pack_path: PathBuf, + + /// The path into which all objects should be written. Commonly '.git/objects' + object_path: Option, + }, + /// Verify the integrity of a pack, index or multi-index file + Verify { + #[clap(flatten)] + args: VerifyOptions, + + /// The '.pack', '.idx' or 'multi-pack-index' file to validate. + path: PathBuf, + }, + } + + #[derive(Debug, clap::Parser)] + pub struct VerifyOptions { + /// output statistical information + #[clap(long, short = 's')] + pub statistics: bool, + /// The algorithm used to verify packs. They differ in costs. + #[clap( + long, + short = 'a', + default_value = "less-time", + possible_values(core::pack::verify::Algorithm::variants()) + )] + pub algorithm: core::pack::verify::Algorithm, + + #[clap(long, conflicts_with("re-encode"))] + /// Decode and parse tags, commits and trees to validate their correctness beyond hashing correctly. + /// + /// Malformed objects should not usually occur, but could be injected on purpose or accident. + /// This will reduce overall performance. + pub decode: bool, + + #[clap(long)] + /// Decode and parse tags, commits and trees to validate their correctness, and re-encode them. + /// + /// This flag is primarily to test the implementation of encoding, and requires to decode the object first. + /// Encoding an object after decoding it should yield exactly the same bytes. + /// This will reduce overall performance even more, as re-encoding requires to transform zero-copy objects into + /// owned objects, causing plenty of allocation to occur. + pub re_encode: bool, + } + + /// + pub mod multi_index { + use std::path::PathBuf; + + #[derive(Debug, clap::Parser)] + pub struct Platform { + /// The path to the index file. + #[clap(short = 'i', long, default_value = ".git/objects/pack/multi-pack-index")] + pub multi_index_path: PathBuf, + + /// Subcommands + #[clap(subcommand)] + pub cmd: Subcommands, + } + + #[derive(Debug, clap::Subcommand)] + pub enum Subcommands { + /// Display all entries of a multi-index: + Entries, + /// Print general information about a multi-index file + Info, + /// Verify a multi-index quickly without inspecting objects themselves + Verify, + /// Create a multi-pack index from one or more pack index files, overwriting possibloy existing files. + Create { + /// Paths to the pack index files to read (with .idx extension). + /// + /// Note for the multi-index to be useful, it should be side-by-side with the supplied `.idx` files. + #[clap(required = true)] + index_paths: Vec, + }, + } + } + + /// + pub mod index { + use std::path::PathBuf; + + use gitoxide_core as core; + + #[derive(Debug, clap::Subcommand)] + pub enum Subcommands { + /// create a pack index from a pack data file. + Create { + /// Specify how to iterate the pack, defaults to 'verify' + /// + /// Valid values are + /// + /// **as-is** do not do anything and expect the pack file to be valid as per the trailing hash, + /// **verify** the input ourselves and validate that it matches with the hash provided in the pack, + /// **restore** hash the input ourselves and ignore failing entries, instead finish the pack with the hash we computed + /// to keep as many objects as possible. + #[clap( + long, + short = 'i', + default_value = "verify", + possible_values(core::pack::index::IterationMode::variants()) + )] + iteration_mode: core::pack::index::IterationMode, + + /// Path to the pack file to read (with .pack extension). + /// + /// If unset, the pack file is expected on stdin. + #[clap(long, short = 'p')] + pack_path: Option, + + /// The folder into which to place the pack and the generated index file + /// + /// If unset, only informational output will be provided to standard output. + directory: Option, + }, + } + } +} + +/// +pub mod mailmap { + use std::path::PathBuf; + + #[derive(Debug, clap::Parser)] + pub struct Platform { + /// The path to the mailmap file. + #[clap(short = 'p', long, default_value = ".mailmap")] + pub path: PathBuf, + + /// Subcommands + #[clap(subcommand)] + pub cmd: Subcommands, + } + + #[derive(Debug, clap::Subcommand)] + pub enum Subcommands { + /// Parse all entries in the mailmap and report malformed lines or collisions. + Verify, + } +} diff --git a/src/plumbing/options/mod.rs b/src/plumbing/options/mod.rs new file mode 100644 index 00000000000..ae2eece8169 --- /dev/null +++ b/src/plumbing/options/mod.rs @@ -0,0 +1,319 @@ +use std::path::PathBuf; + +use git_repository as git; +use git_repository::bstr::BString; +use gitoxide_core as core; + +#[derive(Debug, clap::Parser)] +#[clap(name = "gix-plumbing", about = "The git underworld", version = clap::crate_version!())] +#[clap(subcommand_required = true)] +#[clap(arg_required_else_help = true)] +pub struct Args { + /// The repository to access. + #[clap(short = 'r', long, default_value = ".")] + pub repository: PathBuf, + + /// Add these values to the configuration in the form of `key=value` or `key`. + /// + /// For example, if `key` is `core.abbrev`, set configuration like `[core] abbrev = key`, + /// or `remote.origin.url = foo` to set `[remote "origin"] url = foo`. + #[clap(long, short = 'c', parse(try_from_os_str = git::env::os_str_to_bstring))] + pub config: Vec, + + #[clap(long, short = 't')] + /// The amount of threads to use for some operations. + /// + /// If unset, or the value is 0, there is no limit and all logical cores can be used. + pub threads: Option, + + /// Display verbose messages and progress information + #[clap(long, short = 'v')] + pub verbose: bool, + + /// Bring up a terminal user interface displaying progress visually + #[cfg(feature = "prodash-render-tui")] + #[clap(long, conflicts_with("verbose"))] + pub progress: bool, + + /// The progress TUI will stay up even though the work is already completed. + /// + /// Use this to be able to read progress messages or additional information visible in the TUI log pane. + #[cfg(feature = "prodash-render-tui")] + #[clap(long, conflicts_with("verbose"), requires("progress"))] + pub progress_keep_open: bool, + + /// Determine the format to use when outputting statistics. + #[clap( + long, + short = 'f', + default_value = "human", + possible_values(core::OutputFormat::variants()) + )] + pub format: core::OutputFormat, + + /// The object format to assume when reading files that don't inherently know about it, or when writing files. + #[clap(long, default_value_t = git_repository::hash::Kind::default(), possible_values(&["SHA1"]))] + pub object_hash: git_repository::hash::Kind, + + #[clap(subcommand)] + pub cmd: Subcommands, +} + +#[derive(Debug, clap::Subcommand)] +pub enum Subcommands { + /// Interact with the object database. + #[clap(subcommand)] + Odb(odb::Subcommands), + /// Interact with tree objects. + #[clap(subcommand)] + Tree(tree::Subcommands), + /// Interact with commit objects. + #[clap(subcommand)] + Commit(commit::Subcommands), + /// Verify the integrity of the entire repository + Verify { + #[clap(flatten)] + args: free::pack::VerifyOptions, + }, + /// Query and obtain information about revisions. + #[clap(subcommand)] + Revision(revision::Subcommands), + /// A program just like `git credential`. + #[clap(subcommand)] + Credential(credential::Subcommands), + /// Interact with the mailmap. + #[clap(subcommand)] + Mailmap(mailmap::Subcommands), + /// Interact with the remote hosts. + Remote(remote::Platform), + /// Interact with the exclude files like .gitignore. + #[clap(subcommand)] + Exclude(exclude::Subcommands), + #[clap(subcommand)] + Progress(progress::Subcommands), + Config(config::Platform), + /// Subcommands that need no git repository to run. + #[clap(subcommand)] + Free(free::Subcommands), +} + +pub mod progress { + + #[derive(Debug, clap::Subcommand)] + pub enum Subcommands { + /// Show the implementation progress of gitoxide based on the git configuration that it consumes. + Show, + } +} + +pub mod config { + /// Print all entries in a configuration file or access other sub-commands + #[derive(Debug, clap::Parser)] + #[clap(subcommand_required(false))] + pub struct Platform { + /// The filter terms to limit the output to matching sections and subsections only. + /// + /// Typical filters are `branch` or `remote.origin` or `remote.or*` - git-style globs are supported + /// and comparisons are case-insensitive. + pub filter: Vec, + } +} + +pub mod remote { + use git_repository as git; + + #[derive(Debug, clap::Parser)] + pub struct Platform { + /// The name of the remote to connect to. + /// + /// If unset, the current branch will determine the remote. + #[clap(long, short = 'n')] + pub name: Option, + + /// Connect directly to the given URL, forgoing any configuration from the repository. + #[clap(long, short = 'u', conflicts_with("name"), parse(try_from_os_str = std::convert::TryFrom::try_from))] + pub url: Option, + + /// Output additional typically information provided by the server as part of the connection handshake. + #[clap(long)] + pub handshake_info: bool, + + /// Subcommands + #[clap(subcommand)] + pub cmd: Subcommands, + } + + #[derive(Debug, clap::Subcommand)] + #[clap(visible_alias = "remotes")] + pub enum Subcommands { + /// Print all references available on the remote. + #[cfg(any(feature = "gitoxide-core-async-client", feature = "gitoxide-core-blocking-client"))] + Refs, + /// Print all references available on the remote as filtered through ref-specs. + #[cfg(any(feature = "gitoxide-core-async-client", feature = "gitoxide-core-blocking-client"))] + RefMap { + /// Override the built-in and configured ref-specs with one or more of the given ones. + #[clap(parse(try_from_os_str = git::env::os_str_to_bstring))] + ref_spec: Vec, + }, + } +} + +pub mod mailmap { + #[derive(Debug, clap::Subcommand)] + pub enum Subcommands { + /// Print all entries in configured mailmaps, inform about errors as well. + Entries, + } +} + +pub mod odb { + #[derive(Debug, clap::Subcommand)] + pub enum Subcommands { + /// Print all object names. + Entries, + /// Provide general information about the object database. + Info, + } +} + +pub mod tree { + #[derive(Debug, clap::Subcommand)] + pub enum Subcommands { + /// Print entries in a given tree + Entries { + /// Traverse the entire tree and its subtrees respectively, not only this tree. + #[clap(long, short = 'r')] + recursive: bool, + + /// Provide files size as well. This is expensive as the object is decoded entirely. + #[clap(long, short = 'e')] + extended: bool, + + /// The tree to traverse, or the tree at `HEAD` if unspecified. + treeish: Option, + }, + /// Provide information about a tree. + Info { + /// Provide files size as well. This is expensive as the object is decoded entirely. + #[clap(long, short = 'e')] + extended: bool, + /// The tree to traverse, or the tree at `HEAD` if unspecified. + treeish: Option, + }, + } +} + +pub mod commit { + #[derive(Debug, clap::Subcommand)] + pub enum Subcommands { + /// Describe the current commit or the given one using the name of the closest annotated tag in its ancestry. + Describe { + /// Use annotated tag references only, not all tags. + #[clap(long, short = 't', conflicts_with("all-refs"))] + annotated_tags: bool, + + /// Use all references under the `ref/` namespaces, which includes tag references, local and remote branches. + #[clap(long, short = 'a', conflicts_with("annotated-tags"))] + all_refs: bool, + + /// Only follow the first parent when traversing the commit graph. + #[clap(long, short = 'f')] + first_parent: bool, + + /// Always display the long format, even if that would not be necessary as the id is located directly on a reference. + #[clap(long, short = 'l')] + long: bool, + + /// Consider only the given `n` candidates. This can take longer, but potentially produces more accurate results. + #[clap(long, short = 'c', default_value = "10")] + max_candidates: usize, + + /// Print information on stderr to inform about performance statistics + #[clap(long, short = 's')] + statistics: bool, + + #[clap(long)] + /// If there was no way to describe the commit, fallback to using the abbreviated input revision. + always: bool, + + /// A specification of the revision to use, or the current `HEAD` if unset. + rev_spec: Option, + }, + } +} + +pub mod credential { + #[derive(Debug, clap::Subcommand)] + pub enum Subcommands { + /// Get the credentials fed for `url=` via STDIN. + #[clap(visible_alias = "get")] + Fill, + /// Approve the information piped via STDIN as obtained with last call to `fill` + #[clap(visible_alias = "store")] + Approve, + /// Try to resolve the given revspec and print the object names. + #[clap(visible_alias = "erase")] + Reject, + } +} + +pub mod revision { + #[derive(Debug, clap::Subcommand)] + #[clap(visible_alias = "rev", visible_alias = "r")] + pub enum Subcommands { + /// List all commits reachable from the given rev-spec. + #[clap(visible_alias = "l")] + List { spec: std::ffi::OsString }, + /// Provide the revision specification like `@~1` to explain. + #[clap(visible_alias = "e")] + Explain { spec: std::ffi::OsString }, + /// Try to resolve the given revspec and print the object names. + #[clap(visible_alias = "query", visible_alias = "parse", visible_alias = "p")] + Resolve { + /// Instead of resolving a rev-spec, explain what would be done for the first spec. + /// + /// Equivalent to the `explain` subcommand. + #[clap(short = 'e', long)] + explain: bool, + /// Show the first resulting object similar to how `git cat-file` would, but don't show the resolved spec. + #[clap(short = 'c', long, conflicts_with = "explain")] + cat_file: bool, + /// rev-specs like `@`, `@~1` or `HEAD^2`. + #[clap(required = true)] + specs: Vec, + }, + /// Return the names and hashes of all previously checked-out branches. + #[clap(visible_alias = "prev")] + PreviousBranches, + } +} + +pub mod exclude { + use std::ffi::OsString; + + use git_repository as git; + + #[derive(Debug, clap::Subcommand)] + pub enum Subcommands { + /// Check if path-specs are excluded and print the result similar to `git check-ignore`. + Query { + /// Show actual ignore patterns instead of un-excluding an entry. + /// + /// That way one can understand why an entry might not be excluded. + #[clap(long, short = 'i')] + show_ignore_patterns: bool, + /// Additional patterns to use for exclusions. They have the highest priority. + /// + /// Useful for undoing previous patterns using the '!' prefix. + #[clap(long, short = 'p')] + patterns: Vec, + /// The git path specifications to check for exclusion, or unset to read from stdin one per line. + #[clap(parse(try_from_os_str = std::convert::TryFrom::try_from))] + pathspecs: Vec, + }, + } +} + +/// +pub mod free; diff --git a/src/plumbing/progress.rs b/src/plumbing/progress.rs new file mode 100644 index 00000000000..f3c53871263 --- /dev/null +++ b/src/plumbing/progress.rs @@ -0,0 +1,44 @@ +use crosstermion::crossterm::style::Stylize; +use std::fmt::{Display, Formatter}; + +enum Usage { + InModule(&'static str), +} +use Usage::*; + +impl Display for Usage { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + InModule(m) => write!(f, "mod {m}"), + } + } +} + +struct Record { + config: &'static str, + usage: Usage, + deviation: Option<&'static str>, +} + +static GIT_CONFIG: &[Record] = &[Record { + config: "pack.threads", + usage: InModule("remote::connection::fetch"), + deviation: Some("if unset, it uses all threads as opposed to just 1"), +}]; + +/// A programmatic way to record and display progress. +pub fn show_progress() -> anyhow::Result<()> { + for Record { + config, + usage, + deviation, + } in GIT_CONFIG + { + println!( + "{}: {usage}{}", + config.bold(), + deviation.map(|d| format!(" ({d})")).unwrap_or_default().dark_grey() + ); + } + Ok(()) +} From 317e02a5900189ec7f9f3c2bb27d5696178d7869 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 23 Sep 2022 19:58:19 +0800 Subject: [PATCH 038/113] add support for more types of configurations (#450) --- src/plumbing/progress.rs | 81 ++++++++++++++++++++++++++++++---------- 1 file changed, 62 insertions(+), 19 deletions(-) diff --git a/src/plumbing/progress.rs b/src/plumbing/progress.rs index f3c53871263..ebf4fd654e5 100644 --- a/src/plumbing/progress.rs +++ b/src/plumbing/progress.rs @@ -1,44 +1,87 @@ use crosstermion::crossterm::style::Stylize; +use owo_colors::OwoColorize; use std::fmt::{Display, Formatter}; +#[derive(Clone)] enum Usage { - InModule(&'static str), + NotApplicable, + Planned { + note: Option<&'static str>, + }, + InModule { + name: &'static str, + deviation: Option<&'static str>, + }, } use Usage::*; impl Display for Usage { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { - InModule(m) => write!(f, "mod {m}"), + NotApplicable => f.write_str("not applicable")?, + Planned { note } => { + write!(f, "{}", "planned".blink())?; + if let Some(note) = note { + write!(f, " ℹ {} ℹ", note.bright_white())?; + } + } + InModule { name, deviation } => { + write!(f, "mod {name}")?; + if let Some(deviation) = deviation { + write!(f, "{}", format!(" ❗️{deviation}❗️").bright_white())? + } + } } + Ok(()) } } +impl Usage { + pub fn icon(&self) -> &'static str { + match self { + NotApplicable => "❌", + Planned { .. } => "🕒", + InModule { deviation, .. } => deviation.is_some().then(|| "👌️").unwrap_or("✅"), + } + } +} + +#[derive(Clone)] struct Record { config: &'static str, usage: Usage, - deviation: Option<&'static str>, } -static GIT_CONFIG: &[Record] = &[Record { - config: "pack.threads", - usage: InModule("remote::connection::fetch"), - deviation: Some("if unset, it uses all threads as opposed to just 1"), -}]; +static GIT_CONFIG: &[Record] = &[ + Record { + config: "fetch.output", + usage: NotApplicable, + }, + Record { + config: "fetch.negotiationAlgorithm", + usage: Planned { + note: Some("Implements our own 'naive' algorithm, only"), + }, + }, + Record { + config: "pack.threads", + usage: InModule { + name: "remote::connection::fetch", + deviation: Some("if unset, it uses all threads as opposed to just 1"), + }, + }, +]; /// A programmatic way to record and display progress. pub fn show_progress() -> anyhow::Result<()> { - for Record { - config, - usage, - deviation, - } in GIT_CONFIG - { - println!( - "{}: {usage}{}", - config.bold(), - deviation.map(|d| format!(" ({d})")).unwrap_or_default().dark_grey() - ); + let sorted = { + let mut v: Vec<_> = GIT_CONFIG.into(); + v.sort_by_key(|r| r.config); + v + }; + + for Record { config, usage } in sorted { + println!("{} {}: {usage}", usage.icon(), config.bold(),); } Ok(()) } From b42b08afafd904ff2adb1f00688437357532193a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 23 Sep 2022 20:14:51 +0800 Subject: [PATCH 039/113] refactor (#450) --- README.md | 7 +++---- src/plumbing/main.rs | 7 ++----- src/plumbing/options/mod.rs | 13 ++----------- 3 files changed, 7 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 37c2218f0db..6b6b6ac72d0 100644 --- a/README.md +++ b/README.md @@ -27,10 +27,9 @@ Please see _'Development Status'_ for a listing of all crates and their capabili * Based on the [git-hours] algorithm. * See the [discussion][git-hours-discussion] for some performance data. * **the `gix` program** _(plumbing)_ - lower level commands for use in automation - * **progress** - * [x] **show** - provide an overview of what works and what doesn't from the perspective of the git configuration. - This is likely to change a lot over time depending on actual needs, but maybe useful for you to see - if particular git-configuration is picked up and where it deviates. + * **progress** - provide an overview of what works and what doesn't from the perspective of the git configuration. + This is likely to change a lot over time depending on actual needs, but maybe useful for you to see + if particular git-configuration is picked up and where it deviates. * **config** - list the complete git configuration in human-readable form and optionally filter sections by name. * **exclude** * [x] **query** - check if path specs are excluded via gits exclusion rules like `.gitignore`. diff --git a/src/plumbing/main.rs b/src/plumbing/main.rs index efc8ed7ea85..cdebe387dab 100644 --- a/src/plumbing/main.rs +++ b/src/plumbing/main.rs @@ -15,10 +15,7 @@ use gitoxide_core::pack::verify; use crate::{ plumbing::{ - options::{ - commit, config, credential, exclude, free, mailmap, odb, progress, remote, revision, tree, Args, - Subcommands, - }, + options::{commit, config, credential, exclude, free, mailmap, odb, remote, revision, tree, Args, Subcommands}, show_progress, }, shared::pretty::prepare_and_run, @@ -115,7 +112,7 @@ pub fn main() -> Result<()> { })?; match cmd { - Subcommands::Progress(progress::Subcommands::Show) => show_progress(), + Subcommands::Progress => show_progress(), Subcommands::Credential(cmd) => core::repository::credential( repository(Mode::StrictWithGitInstallConfig)?, match cmd { diff --git a/src/plumbing/options/mod.rs b/src/plumbing/options/mod.rs index ae2eece8169..b7c3ea58db8 100644 --- a/src/plumbing/options/mod.rs +++ b/src/plumbing/options/mod.rs @@ -89,23 +89,14 @@ pub enum Subcommands { /// Interact with the exclude files like .gitignore. #[clap(subcommand)] Exclude(exclude::Subcommands), - #[clap(subcommand)] - Progress(progress::Subcommands), + /// Display overall progress of the gitoxide project as seen from the perspective of git-config. + Progress, Config(config::Platform), /// Subcommands that need no git repository to run. #[clap(subcommand)] Free(free::Subcommands), } -pub mod progress { - - #[derive(Debug, clap::Subcommand)] - pub enum Subcommands { - /// Show the implementation progress of gitoxide based on the git configuration that it consumes. - Show, - } -} - pub mod config { /// Print all entries in a configuration file or access other sub-commands #[derive(Debug, clap::Parser)] From 65e64964c7cd151e53e5a7d4b9ba8fabda1c0e16 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 23 Sep 2022 20:56:35 +0800 Subject: [PATCH 040/113] Add tabled for nicer printing (#450) --- Cargo.lock | 34 ++++++++++++++++++++++++++++++++++ Cargo.toml | 3 ++- src/plumbing/progress.rs | 19 +++++++++++++++---- 3 files changed, 51 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 73196abb6c0..feac5975ea0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -297,6 +297,12 @@ dependencies = [ "utf8-width", ] +[[package]] +name = "bytecount" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c676a478f63e9fa2dd5368a42f28bba0d6c560b775f38583c8bbaa7fcd67c9c" + [[package]] name = "bytes" version = "1.2.1" @@ -954,6 +960,12 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "form_urlencoded" version = "1.1.0" @@ -1837,6 +1849,7 @@ dependencies = [ "gitoxide-core", "owo-colors", "prodash", + "tabled", ] [[package]] @@ -2387,6 +2400,17 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" +[[package]] +name = "papergrid" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "453cf71f2a37af495a1a124bf30d4d7469cfbea58e9f2479be9d222396a518a2" +dependencies = [ + "bytecount", + "fnv", + "unicode-width", +] + [[package]] name = "parking" version = "2.0.0" @@ -2933,6 +2957,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tabled" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5b2f8c37d26d87d2252187b0a45ea3cbf42baca10377c7e7eaaa2800fa9bf97" +dependencies = [ + "papergrid", + "unicode-width", +] + [[package]] name = "tar" version = "0.4.38" diff --git a/Cargo.toml b/Cargo.toml index 96ce65e2e7e..e00a3cae59e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -92,8 +92,9 @@ env_logger = { version = "0.9.0", default-features = false } crosstermion = { version = "0.10.1", optional = true, default-features = false } futures-lite = { version = "1.12.0", optional = true, default-features = false, features = ["std"] } -# for show-progress +# for progress owo-colors = "3.5.0" +tabled = { version = "0.8.0", default-features = false } document-features = { version = "0.2.0", optional = true } diff --git a/src/plumbing/progress.rs b/src/plumbing/progress.rs index ebf4fd654e5..a06ea1e63fc 100644 --- a/src/plumbing/progress.rs +++ b/src/plumbing/progress.rs @@ -1,6 +1,6 @@ -use crosstermion::crossterm::style::Stylize; use owo_colors::OwoColorize; use std::fmt::{Display, Formatter}; +use tabled::{Style, TableIteratorExt, Tabled}; #[derive(Clone)] enum Usage { @@ -52,6 +52,18 @@ struct Record { usage: Usage, } +impl Tabled for Record { + const LENGTH: usize = 3; + + fn fields(&self) -> Vec { + vec![self.usage.icon().into(), self.config.into(), self.usage.to_string()] + } + + fn headers() -> Vec { + vec![] + } +} + static GIT_CONFIG: &[Record] = &[ Record { config: "fetch.output", @@ -80,8 +92,7 @@ pub fn show_progress() -> anyhow::Result<()> { v }; - for Record { config, usage } in sorted { - println!("{} {}: {usage}", usage.icon(), config.bold(),); - } + println!("{}", sorted.table().with(Style::blank())); + println!("\nTotal records: {}", GIT_CONFIG.len()); Ok(()) } From 5c0d0ab66d46a8d093ca0b5451996099a27ef1dd Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 23 Sep 2022 21:03:45 +0800 Subject: [PATCH 041/113] add more records (#450) --- src/plumbing/progress.rs | 53 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/src/plumbing/progress.rs b/src/plumbing/progress.rs index a06ea1e63fc..524e142bc71 100644 --- a/src/plumbing/progress.rs +++ b/src/plumbing/progress.rs @@ -12,12 +12,15 @@ enum Usage { name: &'static str, deviation: Option<&'static str>, }, + /// Needs analysis + Puzzled, } use Usage::*; impl Display for Usage { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { + Puzzled => f.write_str("❓")?, NotApplicable => f.write_str("not applicable")?, Planned { note } => { write!(f, "{}", "planned".blink())?; @@ -39,6 +42,7 @@ impl Display for Usage { impl Usage { pub fn icon(&self) -> &'static str { match self { + Puzzled => "?", NotApplicable => "❌", Planned { .. } => "🕒", InModule { deviation, .. } => deviation.is_some().then(|| "👌️").unwrap_or("✅"), @@ -65,6 +69,48 @@ impl Tabled for Record { } static GIT_CONFIG: &[Record] = &[ + Record { + config: "fetch.recurseSubmodules", + usage: Planned { + note: Some("Seems useful for cargo as well"), + }, + }, + Record { + config: "fetch.fsckObjects", + usage: Puzzled, + }, + Record { + config: "fetch.fsck.", + usage: Puzzled, + }, + Record { + config: "fetch.fsck.skipList", + usage: Puzzled, + }, + Record { + config: "fetch.unpackLimit", + usage: Planned { note: None }, + }, + Record { + config: "fetch.prune", + usage: Planned { note: None }, + }, + Record { + config: "fetch.pruneTags", + usage: Planned { note: None }, + }, + Record { + config: "fetch.writeCommitGraph", + usage: Planned { note: None }, + }, + Record { + config: "fetch.parallel", + usage: Planned { note: None }, + }, + Record { + config: "fetch.showForcedUpdates", + usage: NotApplicable, + }, Record { config: "fetch.output", usage: NotApplicable, @@ -82,6 +128,13 @@ static GIT_CONFIG: &[Record] = &[ deviation: Some("if unset, it uses all threads as opposed to just 1"), }, }, + Record { + config: "pack.indexVersion", + usage: InModule { + name: "remote::connection::fetch", + deviation: None, + }, + }, ]; /// A programmatic way to record and display progress. From 6abd5a4e1daaf91fd109acb714057a82f67fa076 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 23 Sep 2022 21:21:01 +0800 Subject: [PATCH 042/113] complete listing of records based on current usage, probably (#450) --- src/plumbing/progress.rs | 135 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) diff --git a/src/plumbing/progress.rs b/src/plumbing/progress.rs index 524e142bc71..834aa4e348b 100644 --- a/src/plumbing/progress.rs +++ b/src/plumbing/progress.rs @@ -69,6 +69,113 @@ impl Tabled for Record { } static GIT_CONFIG: &[Record] = &[ + Record { + config: "core.bare", + usage: InModule { + name: "config::cache", + deviation: None, + }, + }, + Record { + config: "core.excludesFile", + usage: InModule { + name: "config::cache", + deviation: None, + }, + }, + Record { + config: "core.abbrev", + usage: InModule { + name: "config::cache", + deviation: None, + }, + }, + Record { + config: "core.ignoreCase", + usage: InModule { + name: "config::cache", + deviation: None, + }, + }, + Record { + config: "core.multiPackIndex", + usage: InModule { + name: "config::cache", + deviation: None, + }, + }, + Record { + config: "core.disambiguate", + usage: InModule { + name: "config::cache", + deviation: None, + }, + }, + Record { + config: "core.logAllRefUpdates", + usage: InModule { + name: "config::cache", + deviation: None, + }, + }, + Record { + config: "core.repositoryFormatVersion", + usage: InModule { + name: "config::cache::incubate", + deviation: None, + }, + }, + Record { + config: "extensions.objectFormat", + usage: InModule { + name: "config::cache::incubate", + deviation: Some( + "Support for SHA256 is prepared but not fully implemented yet. For now we abort when encountered.", + ), + }, + }, + Record { + config: "committer.name", + usage: InModule { + name: "repository::identity", + deviation: None, + }, + }, + Record { + config: "committer.email", + usage: InModule { + name: "repository::identity", + deviation: None, + }, + }, + Record { + config: "author.name", + usage: InModule { + name: "repository::identity", + deviation: None, + }, + }, + Record { + config: "author.email", + usage: InModule { + name: "repository::identity", + deviation: None, + }, + }, + Record { + config: "user.name", + usage: InModule { + name: "repository::identity", + deviation: Some("defaults to 'gitoxide'"), + }, + }, + Record { + config: "user.email", + usage: InModule { + name: "repository::identity", + deviation: Some("defaults to 'gitoxide@localhost'"), + }, + }, Record { config: "fetch.recurseSubmodules", usage: Planned { @@ -135,6 +242,34 @@ static GIT_CONFIG: &[Record] = &[ deviation: None, }, }, + Record { + config: "protocol.allow", + usage: InModule { + name: "remote::url::scheme_permission", + deviation: None, + }, + }, + Record { + config: "protocol..allow", + usage: InModule { + name: "remote::url::scheme_permission", + deviation: None, + }, + }, + Record { + config: "url..insteadOf", + usage: InModule { + name: "remote::url::rewrite", + deviation: None, + }, + }, + Record { + config: "url..pushInsteadOf", + usage: InModule { + name: "remote::url::rewrite", + deviation: None, + }, + }, ]; /// A programmatic way to record and display progress. From eade88f8ebc638f504881e8bbbd60d42a5a3d9be Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 23 Sep 2022 21:23:59 +0800 Subject: [PATCH 043/113] slightly nicer styling of config keys (#450) --- src/plumbing/progress.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/plumbing/progress.rs b/src/plumbing/progress.rs index 834aa4e348b..733ccd9d6a9 100644 --- a/src/plumbing/progress.rs +++ b/src/plumbing/progress.rs @@ -1,3 +1,4 @@ +use crosstermion::crossterm::style::Stylize; use owo_colors::OwoColorize; use std::fmt::{Display, Formatter}; use tabled::{Style, TableIteratorExt, Tabled}; @@ -60,7 +61,11 @@ impl Tabled for Record { const LENGTH: usize = 3; fn fields(&self) -> Vec { - vec![self.usage.icon().into(), self.config.into(), self.usage.to_string()] + let mut tokens = self.config.split("."); + let mut buf = vec![tokens.next().expect("present").bold().to_string()]; + buf.extend(tokens.map(ToOwned::to_owned)); + + vec![self.usage.icon().into(), buf.join("."), self.usage.to_string()] } fn headers() -> Vec { From 8dadd70f8b7db1794652805c6238763886a8570d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 23 Sep 2022 21:24:29 +0800 Subject: [PATCH 044/113] thanks clippy --- src/plumbing/progress.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plumbing/progress.rs b/src/plumbing/progress.rs index 733ccd9d6a9..aee757eae33 100644 --- a/src/plumbing/progress.rs +++ b/src/plumbing/progress.rs @@ -61,7 +61,7 @@ impl Tabled for Record { const LENGTH: usize = 3; fn fields(&self) -> Vec { - let mut tokens = self.config.split("."); + let mut tokens = self.config.split('.'); let mut buf = vec![tokens.next().expect("present").bold().to_string()]; buf.extend(tokens.map(ToOwned::to_owned)); From 97a5e972f179c000cec888dcbe4cff13e02d77e5 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 24 Sep 2022 16:16:12 +0800 Subject: [PATCH 045/113] complete pack generation options based on configuration (#450) --- .../src/remote/connection/fetch/mod.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/git-repository/src/remote/connection/fetch/mod.rs b/git-repository/src/remote/connection/fetch/mod.rs index 29fdb0c81ce..5df9e6f0ad9 100644 --- a/git-repository/src/remote/connection/fetch/mod.rs +++ b/git-repository/src/remote/connection/fetch/mod.rs @@ -60,14 +60,14 @@ where git_protocol::fetch::indicate_end_of_interaction(&mut con.transport).ok(); let repo = con.remote.repo; - let _index_version = config::pack_index_version(repo)?; - let _thread_limit = config::index_threads(repo)?; - // let options = git_pack::bundle::write::Options { - // thread_limit: ctx.thread_limit, - // index_version: git_pack::index::Version::V2, - // iteration_mode: git_pack::data::input::Mode::Verify, - // object_hash: ctx.object_hash, - // }; + let index_version = config::pack_index_version(repo)?; + let thread_limit = config::index_threads(repo)?; + let _options = git_pack::bundle::write::Options { + thread_limit, + index_version, + iteration_mode: git_pack::data::input::Mode::Verify, + object_hash: con.remote.repo.object_hash(), + }; todo!() } From 5e93ef53e43c7ce1e5f964d792ff97b426802b4a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 24 Sep 2022 18:42:51 +0800 Subject: [PATCH 046/113] refactor (#450) --- git-repository/src/config/cache/init.rs | 24 +++++------------------- git-repository/src/config/cache/util.rs | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/git-repository/src/config/cache/init.rs b/git-repository/src/config/cache/init.rs index c9c03d426ff..1c368cbfcdb 100644 --- a/git-repository/src/config/cache/init.rs +++ b/git-repository/src/config/cache/init.rs @@ -1,7 +1,7 @@ use std::path::PathBuf; use super::{interpolate_context, util, Error, StageOne}; -use crate::{config::Cache, repository, revision::spec::parse::ObjectKindHint}; +use crate::{config::Cache, repository}; /// Initialization impl Cache { @@ -129,28 +129,14 @@ impl Cache { Err(err) => return Err(err), }; - use util::config_bool; - let reflog = util::query_refupdates(&config); - let ignore_case = config_bool(&config, "core.ignoreCase", false, lenient_config)?; - let use_multi_pack_index = config_bool(&config, "core.multiPackIndex", true, lenient_config)?; - let object_kind_hint = config.string("core", None, "disambiguate").and_then(|value| { - Some(match value.as_ref().as_ref() { - b"commit" => ObjectKindHint::Commit, - b"committish" => ObjectKindHint::Committish, - b"tree" => ObjectKindHint::Tree, - b"treeish" => ObjectKindHint::Treeish, - b"blob" => ObjectKindHint::Blob, - _ => return None, - }) - }); Ok(Cache { resolved: config.into(), - use_multi_pack_index, + use_multi_pack_index: util::config_bool(&config, "core.multiPackIndex", true, lenient_config)?, object_hash, - object_kind_hint, - reflog, + object_kind_hint: util::disambiguate_hint(&config), + reflog: util::query_refupdates(&config), is_bare, - ignore_case, + ignore_case: util::config_bool(&config, "core.ignoreCase", false, lenient_config)?, hex_len, filter_config_section, excludes_file, diff --git a/git-repository/src/config/cache/util.rs b/git-repository/src/config/cache/util.rs index a440cf8d58e..aabecfa2674 100644 --- a/git-repository/src/config/cache/util.rs +++ b/git-repository/src/config/cache/util.rs @@ -2,6 +2,7 @@ use std::convert::TryFrom; use super::Error; use crate::bstr::ByteSlice; +use crate::revision::spec::parse::ObjectKindHint; pub(crate) fn interpolate_context<'a>( git_install_dir: Option<&'a std::path::Path>, @@ -93,3 +94,16 @@ pub(crate) fn parse_core_abbrev( None => Ok(None), } } + +pub(crate) fn disambiguate_hint(config: &git_configFile) -> Option { + config.string("core", None, "disambiguate").and_then(|value| { + Some(match value.as_ref().as_ref() { + b"commit" => ObjectKindHint::Commit, + b"committish" => ObjectKindHint::Committish, + b"tree" => ObjectKindHint::Tree, + b"treeish" => ObjectKindHint::Treeish, + b"blob" => ObjectKindHint::Blob, + _ => return None, + }) + }) +} From 31a7089f2583832727e2175ada6fb5c30c3beebe Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 24 Sep 2022 22:43:15 +0800 Subject: [PATCH 047/113] feat: make some private methods public to give callers more flexibility. (#450) This allows to implement the fetch-negotiation part oneself and break free from constraints of the delegate. --- git-protocol/src/fetch/arguments/blocking_io.rs | 3 ++- git-protocol/src/fetch/arguments/mod.rs | 4 +++- git-protocol/src/fetch/command.rs | 5 +++-- git-repository/src/remote/connection/fetch/negotiate.rs | 0 4 files changed, 8 insertions(+), 4 deletions(-) create mode 100644 git-repository/src/remote/connection/fetch/negotiate.rs diff --git a/git-protocol/src/fetch/arguments/blocking_io.rs b/git-protocol/src/fetch/arguments/blocking_io.rs index dac2163942d..98ac93f75b2 100644 --- a/git-protocol/src/fetch/arguments/blocking_io.rs +++ b/git-protocol/src/fetch/arguments/blocking_io.rs @@ -5,7 +5,8 @@ use git_transport::{client, client::TransportV2Ext}; use crate::fetch::{Arguments, Command}; impl Arguments { - pub(crate) fn send<'a, T: client::Transport + 'a>( + /// Send fetch arguments to the server, and indicate this is the end of negotiations only if `add_done_argument` is present. + pub fn send<'a, T: client::Transport + 'a>( &mut self, transport: &'a mut T, add_done_argument: bool, diff --git a/git-protocol/src/fetch/arguments/mod.rs b/git-protocol/src/fetch/arguments/mod.rs index 0539c7eca34..5a4bac278e9 100644 --- a/git-protocol/src/fetch/arguments/mod.rs +++ b/git-protocol/src/fetch/arguments/mod.rs @@ -125,8 +125,10 @@ impl Arguments { fn prefixed(&mut self, prefix: &str, value: impl fmt::Display) { self.args.push(format!("{}{}", prefix, value).into()); } + /// Create a new instance to help setting up arguments to send to the server as part of a `fetch` operation + /// for which `features` are the available and configured features to use. #[cfg(any(feature = "async-client", feature = "blocking-client"))] - pub(crate) fn new(version: git_transport::Protocol, features: Vec) -> Self { + pub fn new(version: git_transport::Protocol, features: Vec) -> Self { use crate::fetch::Command; let has = |name: &str| features.iter().any(|f| f.0 == name); let filter = has("filter"); diff --git a/git-protocol/src/fetch/command.rs b/git-protocol/src/fetch/command.rs index 162134d8eba..3efa2fddc0c 100644 --- a/git-protocol/src/fetch/command.rs +++ b/git-protocol/src/fetch/command.rs @@ -112,8 +112,9 @@ mod with_io { } } - /// Turns on all modern features for V1 and all supported features for V2. - pub(crate) fn default_features( + /// Turns on all modern features for V1 and all supported features for V2, returning them as a vector of features. + /// Note that this is the basis for any fetch operation as these features fulfil basic requirements and reasonably up-to-date servers. + pub fn default_features( &self, version: git_transport::Protocol, server_capabilities: &Capabilities, diff --git a/git-repository/src/remote/connection/fetch/negotiate.rs b/git-repository/src/remote/connection/fetch/negotiate.rs new file mode 100644 index 00000000000..e69de29bb2d From 3c188b2253bfb6d47394718425eef2d1a0547949 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 24 Sep 2022 22:44:42 +0800 Subject: [PATCH 048/113] Add remotes. as planned feature for remotes (#450) --- src/plumbing/progress.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/plumbing/progress.rs b/src/plumbing/progress.rs index aee757eae33..b49de4c00be 100644 --- a/src/plumbing/progress.rs +++ b/src/plumbing/progress.rs @@ -261,6 +261,12 @@ static GIT_CONFIG: &[Record] = &[ deviation: None, }, }, + Record { + config: "remotes.", + usage: Planned { + note: Some("useful for multi-remote fetches as part of the standard API, maybe just `group(name) -> Option>`"), + }, + }, Record { config: "url..insteadOf", usage: InModule { From 4997e5616c39f3d3be74f289c25080d9898b28f5 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 24 Sep 2022 22:45:03 +0800 Subject: [PATCH 049/113] port part of the negotation logic over, but a lot is still missing (#450) --- git-repository/src/config/cache/init.rs | 13 ++- git-repository/src/config/cache/util.rs | 2 +- .../src/remote/connection/fetch/mod.rs | 85 +++++++++++++++++-- .../src/remote/connection/fetch/negotiate.rs | 24 ++++++ 4 files changed, 112 insertions(+), 12 deletions(-) diff --git a/git-repository/src/config/cache/init.rs b/git-repository/src/config/cache/init.rs index 1c368cbfcdb..15e9a3e4507 100644 --- a/git-repository/src/config/cache/init.rs +++ b/git-repository/src/config/cache/init.rs @@ -129,14 +129,19 @@ impl Cache { Err(err) => return Err(err), }; + use util::config_bool; + let reflog = util::query_refupdates(&config); + let ignore_case = config_bool(&config, "core.ignoreCase", false, lenient_config)?; + let use_multi_pack_index = config_bool(&config, "core.multiPackIndex", true, lenient_config)?; + let object_kind_hint = util::disambiguate_hint(&config); Ok(Cache { resolved: config.into(), - use_multi_pack_index: util::config_bool(&config, "core.multiPackIndex", true, lenient_config)?, + use_multi_pack_index, object_hash, - object_kind_hint: util::disambiguate_hint(&config), - reflog: util::query_refupdates(&config), + object_kind_hint, + reflog, is_bare, - ignore_case: util::config_bool(&config, "core.ignoreCase", false, lenient_config)?, + ignore_case, hex_len, filter_config_section, excludes_file, diff --git a/git-repository/src/config/cache/util.rs b/git-repository/src/config/cache/util.rs index aabecfa2674..9c81b5db74a 100644 --- a/git-repository/src/config/cache/util.rs +++ b/git-repository/src/config/cache/util.rs @@ -95,7 +95,7 @@ pub(crate) fn parse_core_abbrev( } } -pub(crate) fn disambiguate_hint(config: &git_configFile) -> Option { +pub(crate) fn disambiguate_hint(config: &git_config::File<'static>) -> Option { config.string("core", None, "disambiguate").and_then(|value| { Some(match value.as_ref().as_ref() { b"commit" => ObjectKindHint::Commit, diff --git a/git-repository/src/remote/connection/fetch/mod.rs b/git-repository/src/remote/connection/fetch/mod.rs index 5df9e6f0ad9..95cadfb49c2 100644 --- a/git-repository/src/remote/connection/fetch/mod.rs +++ b/git-repository/src/remote/connection/fetch/mod.rs @@ -7,7 +7,6 @@ use std::sync::atomic::AtomicBool; mod error { /// The error returned by [`receive()`](super::Prepare::receive()). #[derive(Debug, thiserror::Error)] - #[error("TBD")] pub enum Error { #[error("{message}{}", desired.map(|n| format!(" (got {})", n)).unwrap_or_default())] Configuration { @@ -15,10 +14,19 @@ mod error { desired: Option, source: Option, }, + #[error(transparent)] + FetchResponse(#[from] git_protocol::fetch::response::Error), + #[error(transparent)] + Negotiate(#[from] super::negotiate::Error), + #[error(transparent)] + Client(#[from] git_protocol::transport::client::Error), } } pub use error::Error; +/// +pub mod negotiate; + impl<'remote, 'repo, T, P> Connection<'remote, 'repo, T, P> where T: Transport, @@ -57,22 +65,85 @@ where /// experimented with. We currently implement something we could call 'naive' which works for now. pub fn receive(mut self, _should_interrupt: &AtomicBool) -> Result<(), Error> { let mut con = self.con.take().expect("receive() can only be called once"); - git_protocol::fetch::indicate_end_of_interaction(&mut con.transport).ok(); + let handshake = &self.ref_map.handshake; + let protocol_version = handshake.server_protocol_version; + + let fetch = git_protocol::fetch::Command::Fetch; + let fetch_features = fetch.default_features(protocol_version, &handshake.capabilities); + + git_protocol::fetch::Response::check_required_features(protocol_version, &fetch_features)?; + let sideband_all = fetch_features.iter().any(|(n, _)| *n == "sideband-all"); + let mut arguments = git_protocol::fetch::Arguments::new(protocol_version, fetch_features); + let mut previous_response = None::; + let mut round = 1; + let progress = &mut con.progress; let repo = con.remote.repo; - let index_version = config::pack_index_version(repo)?; - let thread_limit = config::index_threads(repo)?; + + let _reader = 'negotiation: loop { + progress.step(); + progress.set_name(format!("negotiate (round {})", round)); + let is_done = match negotiate::one_round( + negotiate::Algorithm::Naive, + round, + repo, + &self.ref_map, + &mut arguments, + previous_response.as_ref(), + ) { + Ok(is_done) => is_done, + Err(err) => { + git_protocol::fetch::indicate_end_of_interaction(&mut con.transport).ok(); + return Err(err.into()); + } + }; + round += 1; + let mut reader = arguments.send(&mut con.transport, is_done)?; + if sideband_all { + setup_remote_progress(progress, &mut reader); + } + let response = git_protocol::fetch::Response::from_line_reader(protocol_version, &mut reader)?; + if response.has_pack() { + progress.step(); + progress.set_name("receiving pack"); + if !sideband_all { + setup_remote_progress(progress, &mut reader); + } + break 'negotiation reader; + } else { + previous_response = Some(response); + } + }; + let _options = git_pack::bundle::write::Options { - thread_limit, - index_version, + thread_limit: config::index_threads(repo)?, + index_version: config::pack_index_version(repo)?, iteration_mode: git_pack::data::input::Mode::Verify, object_hash: con.remote.repo.object_hash(), }; + // git_pack::Bundle::write_to_directory(); + todo!("read pack"); - todo!() + if matches!(protocol_version, git_protocol::transport::Protocol::V2) { + git_protocol::fetch::indicate_end_of_interaction(&mut con.transport).ok(); + } + todo!("apply refs") } } +fn setup_remote_progress( + progress: &mut impl Progress, + reader: &mut Box, +) { + use git_protocol::transport::client::ExtendedBufRead; + reader.set_progress_handler(Some(Box::new({ + let mut remote_progress = progress.add_child("remote"); + move |is_err: bool, data: &[u8]| { + git_protocol::RemoteProgress::translate_to_progress(is_err, data, &mut remote_progress) + } + }) as git_protocol::transport::client::HandleProgress)); +} + mod config; /// A structure to hold the result of the handshake with the remote and configure the upcoming fetch operation. diff --git a/git-repository/src/remote/connection/fetch/negotiate.rs b/git-repository/src/remote/connection/fetch/negotiate.rs index e69de29bb2d..77ed8b78712 100644 --- a/git-repository/src/remote/connection/fetch/negotiate.rs +++ b/git-repository/src/remote/connection/fetch/negotiate.rs @@ -0,0 +1,24 @@ +#[derive(Copy, Clone)] +pub(crate) enum Algorithm { + /// Our very own implementation that probably should be replaced by one of the known algorithms soon. + Naive, +} + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("We were unable to figure out what objects the server should send after {rounds} round(s)")] + NegotiationFailed { rounds: usize }, +} + +/// Negotiate one round with `algo` by looking at `ref_map` and adjust `arguments` to contain the haves and wants. +/// If this is not the first round, the `previous_response` is set with the last recorded server response. +pub(crate) fn one_round( + _algo: Algorithm, + _round: usize, + _repo: &crate::Repository, + _ref_map: &crate::remote::fetch::RefMap<'_>, + _arguments: &mut git_protocol::fetch::Arguments, + _previous_response: Option<&git_protocol::fetch::Response>, +) -> Result { + todo!() +} From 485020252da95b1369326156ebd8ff6052f591ec Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 25 Sep 2022 11:59:16 +0800 Subject: [PATCH 050/113] Improve docs slightly (#450) --- git-ref/src/target.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git-ref/src/target.rs b/git-ref/src/target.rs index 8e38d75a1fd..a45c2ddd3b5 100644 --- a/git-ref/src/target.rs +++ b/git-ref/src/target.rs @@ -19,7 +19,7 @@ impl<'a> TargetRef<'a> { TargetRef::Peeled(oid) => Some(oid), } } - /// Interpret this target as object id or panic if it is symbolic. + /// Interpret this target as object id or **panic** if it is symbolic. pub fn id(&self) -> &oid { match self { TargetRef::Symbolic(_) => panic!("BUG: tries to obtain object id from symbolic target"), From 0b6ed60f842f0a36f61f187651080540a358758e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 25 Sep 2022 11:59:44 +0800 Subject: [PATCH 051/113] fix: `bundle::write::Error` is now publicly available (#450) --- git-pack/src/bundle/write/error.rs | 2 ++ git-pack/src/bundle/write/mod.rs | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/git-pack/src/bundle/write/error.rs b/git-pack/src/bundle/write/error.rs index 002fd0170a7..2a967b75b21 100644 --- a/git-pack/src/bundle/write/error.rs +++ b/git-pack/src/bundle/write/error.rs @@ -2,7 +2,9 @@ use std::io; use git_tempfile::handle::Writable; +/// The error returned by [`Bundle::write_to_directory()`][crate::Bundle::write_to_directory()] #[derive(thiserror::Error, Debug)] +#[allow(missing_docs)] pub enum Error { #[error("An IO error occurred when reading the pack or creating a temporary file")] Io(#[from] io::Error), diff --git a/git-pack/src/bundle/write/mod.rs b/git-pack/src/bundle/write/mod.rs index 5d389fec9cc..ecc18c4e4d2 100644 --- a/git-pack/src/bundle/write/mod.rs +++ b/git-pack/src/bundle/write/mod.rs @@ -10,7 +10,7 @@ use git_tempfile::{handle::Writable, AutoRemove, ContainingDirectory}; use crate::data; mod error; -use error::Error; +pub use error::Error; mod types; use types::{LockWriter, PassThrough}; From 0bcb2fd4d5d8692a0e91c70bcbcbff9dff60442f Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 25 Sep 2022 12:01:07 +0800 Subject: [PATCH 052/113] feat: `fetch::Arguments::is_empty()` to help decide if arguments should be sent at all. (#450) Furthermore, the `Debug` trait was added. --- git-protocol/src/fetch/arguments/mod.rs | 8 ++++++++ git-protocol/src/fetch/refs/mod.rs | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/git-protocol/src/fetch/arguments/mod.rs b/git-protocol/src/fetch/arguments/mod.rs index 5a4bac278e9..6a0ebda438e 100644 --- a/git-protocol/src/fetch/arguments/mod.rs +++ b/git-protocol/src/fetch/arguments/mod.rs @@ -3,6 +3,7 @@ use std::fmt; use bstr::{BStr, BString, ByteVec}; /// The arguments passed to a server command. +#[derive(Debug)] pub struct Arguments { /// The active features/capabilities of the fetch invocation #[cfg(any(feature = "async-client", feature = "blocking-client"))] @@ -24,6 +25,13 @@ pub struct Arguments { } impl Arguments { + /// Return true if there is no argument at all. + /// + /// This can happen if callers assure that they won't add 'wants' if their 'have' is the same, i.e. if the remote has nothing + /// new for them. + pub fn is_empty(&self) -> bool { + self.args.is_empty() + } /// Return true if ref filters is supported. pub fn can_use_filter(&self) -> bool { self.filter diff --git a/git-protocol/src/fetch/refs/mod.rs b/git-protocol/src/fetch/refs/mod.rs index ba3fafd2471..122df7f644b 100644 --- a/git-protocol/src/fetch/refs/mod.rs +++ b/git-protocol/src/fetch/refs/mod.rs @@ -50,7 +50,7 @@ pub mod parse { pub enum Ref { /// A ref pointing to a `tag` object, which in turns points to an `object`, usually a commit Peeled { - /// The name at which the ref is located, like `refs/heads/main`. + /// The name at which the ref is located, like `refs/tags/1.0`. full_ref_name: BString, /// The hash of the tag the ref points to. tag: git_hash::ObjectId, From aad17ba50f8c77465004a00da2146a87fc770646 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 25 Sep 2022 12:01:43 +0800 Subject: [PATCH 053/113] A first test for validating nothing-new is a no-op (#450) --- .../src/remote/connection/fetch/mod.rs | 40 ++++++++++++++++--- .../src/remote/connection/fetch/negotiate.rs | 30 +++++++++++--- git-repository/src/remote/mod.rs | 10 +++++ git-repository/tests/remote/fetch.rs | 7 ++++ 4 files changed, 75 insertions(+), 12 deletions(-) diff --git a/git-repository/src/remote/connection/fetch/mod.rs b/git-repository/src/remote/connection/fetch/mod.rs index 95cadfb49c2..ac594027d31 100644 --- a/git-repository/src/remote/connection/fetch/mod.rs +++ b/git-repository/src/remote/connection/fetch/mod.rs @@ -1,6 +1,7 @@ use crate::remote::fetch::RefMap; use crate::remote::{ref_map, Connection}; use crate::Progress; +use git_odb::FindExt; use git_protocol::transport::client::Transport; use std::sync::atomic::AtomicBool; @@ -20,10 +21,20 @@ mod error { Negotiate(#[from] super::negotiate::Error), #[error(transparent)] Client(#[from] git_protocol::transport::client::Error), + #[error(transparent)] + WritePack(#[from] git_pack::bundle::write::Error), } } pub use error::Error; +/// The outcome of receiving a pack via [`Prepare::receive()`]. +#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)] +#[cfg_attr(feature = "serde1", derive(serde::Serialize, serde::Deserialize))] +pub struct Outcome { + /// Information collected while writing the pack and its index. + pub write_pack_bundle: git_pack::bundle::write::Outcome, +} + /// pub mod negotiate; @@ -58,12 +69,14 @@ where P: Progress, { /// Receive the pack and perform the operation as configured by git via `git-config` or overridden by various builder methods. + /// Return `Ok(None)` if there was nothing to do because all remote refs are at the same state as they are locally, or `Ok(Some(outcome))` + /// to inform about all the changes that were made. /// /// ### Negotiation /// /// "fetch.negotiationAlgorithm" describes algorithms `git` uses currently, with the default being `consecutive` and `skipping` being /// experimented with. We currently implement something we could call 'naive' which works for now. - pub fn receive(mut self, _should_interrupt: &AtomicBool) -> Result<(), Error> { + pub fn receive(mut self, should_interrupt: &AtomicBool) -> Result, Error> { let mut con = self.con.take().expect("receive() can only be called once"); let handshake = &self.ref_map.handshake; @@ -80,7 +93,7 @@ where let progress = &mut con.progress; let repo = con.remote.repo; - let _reader = 'negotiation: loop { + let reader = 'negotiation: loop { progress.step(); progress.set_name(format!("negotiate (round {})", round)); let is_done = match negotiate::one_round( @@ -91,6 +104,10 @@ where &mut arguments, previous_response.as_ref(), ) { + Ok(_) if arguments.is_empty() => { + git_protocol::fetch::indicate_end_of_interaction(&mut con.transport).ok(); + return Ok(None); + } Ok(is_done) => is_done, Err(err) => { git_protocol::fetch::indicate_end_of_interaction(&mut con.transport).ok(); @@ -115,19 +132,30 @@ where } }; - let _options = git_pack::bundle::write::Options { + let options = git_pack::bundle::write::Options { thread_limit: config::index_threads(repo)?, index_version: config::pack_index_version(repo)?, iteration_mode: git_pack::data::input::Mode::Verify, object_hash: con.remote.repo.object_hash(), }; - // git_pack::Bundle::write_to_directory(); - todo!("read pack"); + let write_pack_bundle = git_pack::Bundle::write_to_directory( + reader, + Some(repo.objects.store_ref().path().join("pack")), + con.progress, + should_interrupt, + Some(Box::new({ + let repo = repo.clone(); + move |oid, buf| repo.objects.find(oid, buf).ok() + })), + options, + )?; if matches!(protocol_version, git_protocol::transport::Protocol::V2) { git_protocol::fetch::indicate_end_of_interaction(&mut con.transport).ok(); } - todo!("apply refs") + + // TODO: apply refs + Ok(Some(Outcome { write_pack_bundle })) } } diff --git a/git-repository/src/remote/connection/fetch/negotiate.rs b/git-repository/src/remote/connection/fetch/negotiate.rs index 77ed8b78712..7aad05a8e1d 100644 --- a/git-repository/src/remote/connection/fetch/negotiate.rs +++ b/git-repository/src/remote/connection/fetch/negotiate.rs @@ -12,13 +12,31 @@ pub enum Error { /// Negotiate one round with `algo` by looking at `ref_map` and adjust `arguments` to contain the haves and wants. /// If this is not the first round, the `previous_response` is set with the last recorded server response. +/// Returns `true` if the negotiation is done from our side so the server won't keep asking. pub(crate) fn one_round( - _algo: Algorithm, - _round: usize, - _repo: &crate::Repository, - _ref_map: &crate::remote::fetch::RefMap<'_>, - _arguments: &mut git_protocol::fetch::Arguments, + algo: Algorithm, + round: usize, + repo: &crate::Repository, + ref_map: &crate::remote::fetch::RefMap<'_>, + arguments: &mut git_protocol::fetch::Arguments, _previous_response: Option<&git_protocol::fetch::Response>, ) -> Result { - todo!() + match algo { + Algorithm::Naive => { + assert_eq!(round, 1, "Naive always finishes after the first round, and claims."); + for mapping in &ref_map.mappings { + if let Some(have_id) = mapping.local.as_ref().and_then(|name| { + repo.find_reference(name) + .ok() + .and_then(|r| r.target().try_id().map(ToOwned::to_owned)) + }) { + if mapping.remote.as_id() != have_id { + arguments.want(mapping.remote.as_id()); + arguments.have(have_id); + } + } + } + Ok(true) + } + } } diff --git a/git-repository/src/remote/mod.rs b/git-repository/src/remote/mod.rs index 1635f23ad80..4be4fa4dea0 100644 --- a/git-repository/src/remote/mod.rs +++ b/git-repository/src/remote/mod.rs @@ -54,6 +54,16 @@ pub mod fetch { Ref(git_protocol::fetch::Ref), } + impl Source { + /// Return either the direct object id we refer to or the direct target that a reference refers to. + pub fn as_id(&self) -> &git_hash::oid { + match self { + Source::ObjectId(id) => id, + Source::Ref(r) => r.unpack().1, + } + } + } + /// A mapping between a single remote reference and its advertised objects to a local destination which may or may not exist. #[derive(Debug, Clone)] pub struct Mapping { diff --git a/git-repository/tests/remote/fetch.rs b/git-repository/tests/remote/fetch.rs index 3150fcb04a1..1483854158f 100644 --- a/git-repository/tests/remote/fetch.rs +++ b/git-repository/tests/remote/fetch.rs @@ -3,6 +3,7 @@ mod blocking_io { use git_features::progress; use git_repository as git; use git_repository::remote::Direction::Fetch; + use std::sync::atomic::AtomicBool; use crate::remote; @@ -30,6 +31,12 @@ mod blocking_io { .prepare_fetch(Default::default())?; // early drops are fine and won't block. } + // TODO: make sure there is actually something to do to run into other issues + let outcome = remote + .connect(Fetch, progress::Discard)? + .prepare_fetch(Default::default())? + .receive(&AtomicBool::default())?; + assert_eq!(outcome, None, "there is nothing to do right after a clone."); } Ok(()) } From 72ce7fdced10b8359e74daea3bb35ab73b29e7c0 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 25 Sep 2022 12:17:24 +0800 Subject: [PATCH 054/113] add test to show that empty packs won't be written as expected behaviour. (#450) Even though it's possible, so maybe one day we lift this. This test probably doesn't realize that no objects are written. --- git-pack/src/index/write/error.rs | 2 +- git-pack/src/index/write/mod.rs | 9 ++------- .../tests/pack/data/output/count_and_entries.rs | 15 +++++++++++++++ 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/git-pack/src/index/write/error.rs b/git-pack/src/index/write/error.rs index cd022aefa01..7195b859c43 100644 --- a/git-pack/src/index/write/error.rs +++ b/git-pack/src/index/write/error.rs @@ -14,7 +14,7 @@ pub enum Error { IteratorInvariantNoRefDelta, #[error("The iterator failed to set a trailing hash over all prior pack entries in the last provided entry")] IteratorInvariantTrailer, - #[error("Did not encounter a single base")] + #[error("Did not encounter a single base - refusing to write empty pack.")] IteratorInvariantBasesPresent, #[error("Only u32::MAX objects can be stored in a pack, found {0}")] IteratorInvariantTooManyObjects(usize), diff --git a/git-pack/src/index/write/mod.rs b/git-pack/src/index/write/mod.rs index 9ea8cfda3b5..5b4f2f6a507 100644 --- a/git-pack/src/index/write/mod.rs +++ b/git-pack/src/index/write/mod.rs @@ -171,7 +171,7 @@ impl crate::index::File { modify_base(data, entry, bytes, kind.hash()); Ok::<_, Error>(()) }, - crate::cache::delta::traverse::Options { + traverse::Options { object_progress: root_progress.add_child("Resolving"), size_progress: root_progress.add_child("Decoding"), thread_limit, @@ -215,12 +215,7 @@ impl crate::index::File { } } -fn modify_base( - entry: &mut crate::index::write::TreeEntry, - pack_entry: &crate::data::Entry, - decompressed: &[u8], - hash: git_hash::Kind, -) { +fn modify_base(entry: &mut TreeEntry, pack_entry: &crate::data::Entry, decompressed: &[u8], hash: git_hash::Kind) { fn compute_hash(kind: git_object::Kind, bytes: &[u8], object_hash: git_hash::Kind) -> git_hash::ObjectId { let mut hasher = git_features::hash::hasher(object_hash); hasher.update(&git_object::encode::loose_header(kind, bytes.len())); diff --git a/git-pack/tests/pack/data/output/count_and_entries.rs b/git-pack/tests/pack/data/output/count_and_entries.rs index a5fca708dc7..d08325c3474 100644 --- a/git-pack/tests/pack/data/output/count_and_entries.rs +++ b/git-pack/tests/pack/data/output/count_and_entries.rs @@ -321,6 +321,21 @@ fn traversals() -> crate::Result { Ok(()) } +#[test] +fn empty_pack_is_not_allowed() { + assert_eq!( + write_and_verify( + db(DbKind::DeterministicGeneratedContent).unwrap(), + vec![], + hex_to_id("029d08823bd8a8eab510ad6ac75c823cfd3ed31e"), + None, + ) + .unwrap_err() + .to_string(), + "Did not encounter a single base - refusing to write empty pack." + ); +} + fn write_and_verify( db: git_odb::HandleArc, entries: Vec, From 4d908150c12b90a70e71f1ca6541109dbacb68fe Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 25 Sep 2022 15:16:53 +0800 Subject: [PATCH 055/113] allow git-repository to grow (#450) --- etc/check-package-size.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/check-package-size.sh b/etc/check-package-size.sh index b4b1e6572dc..f10113c73ee 100755 --- a/etc/check-package-size.sh +++ b/etc/check-package-size.sh @@ -57,6 +57,6 @@ echo "in root: gitoxide CLI" (enter git-odb && indent cargo diet -n --package-size-limit 120KB) (enter git-protocol && indent cargo diet -n --package-size-limit 50KB) (enter git-packetline && indent cargo diet -n --package-size-limit 35KB) -(enter git-repository && indent cargo diet -n --package-size-limit 175KB) +(enter git-repository && indent cargo diet -n --package-size-limit 185KB) (enter git-transport && indent cargo diet -n --package-size-limit 55KB) (enter gitoxide-core && indent cargo diet -n --package-size-limit 90KB) From e05c1fefeed23dbedf0420e04a2a408510775380 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 25 Sep 2022 15:33:20 +0800 Subject: [PATCH 056/113] feat: Allow defaulting `client::Capabilities`. (#450) This can be useful in conjunction with `std::mem::take()`, even though an empty data structure like that doesn't bear any significance beyond that. --- git-transport/src/client/capabilities.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git-transport/src/client/capabilities.rs b/git-transport/src/client/capabilities.rs index f6e1efb1513..241bf11b3f7 100644 --- a/git-transport/src/client/capabilities.rs +++ b/git-transport/src/client/capabilities.rs @@ -23,7 +23,7 @@ pub enum Error { } /// A structure to represent multiple [capabilities][Capability] or features supported by the server. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] #[cfg_attr(feature = "serde1", derive(serde::Serialize, serde::Deserialize))] pub struct Capabilities { data: BString, From aed93d22d7a7faf4d60f1f603830c882175cbcb6 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 25 Sep 2022 15:34:31 +0800 Subject: [PATCH 057/113] feat: Allow defaulting `fetch::handshake::Outcome`. (#450) This is useful in conjunction with `std::mem::take()`. --- git-protocol/src/fetch/handshake.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git-protocol/src/fetch/handshake.rs b/git-protocol/src/fetch/handshake.rs index 78d71095339..46e313cc6d8 100644 --- a/git-protocol/src/fetch/handshake.rs +++ b/git-protocol/src/fetch/handshake.rs @@ -3,7 +3,7 @@ use git_transport::client::Capabilities; use crate::fetch::Ref; /// The result of the [`handshake()`][super::handshake()] function. -#[derive(Debug, Clone)] +#[derive(Default, Debug, Clone)] #[cfg_attr(feature = "serde1", derive(serde::Serialize, serde::Deserialize))] pub struct Outcome { /// The protocol version the server responded with. It might have downgraded the desired version. From 5f73b257c551ef899c9e34dd5772654d51444d8b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 25 Sep 2022 15:35:04 +0800 Subject: [PATCH 058/113] Don't degenerate information in case there is no update needed. (#450) That way the caller will always have the information we gathered expensively in the process, whether or not they want to use them. --- .../src/remote/connection/fetch/mod.rs | 36 ++++++++++++------- git-repository/src/remote/mod.rs | 23 +++++++++++- git-repository/tests/remote/fetch.rs | 2 +- 3 files changed, 47 insertions(+), 14 deletions(-) diff --git a/git-repository/src/remote/connection/fetch/mod.rs b/git-repository/src/remote/connection/fetch/mod.rs index ac594027d31..f81dfa4d0db 100644 --- a/git-repository/src/remote/connection/fetch/mod.rs +++ b/git-repository/src/remote/connection/fetch/mod.rs @@ -1,4 +1,4 @@ -use crate::remote::fetch::RefMap; +use crate::remote::fetch::{Outcome, RefMap, Status}; use crate::remote::{ref_map, Connection}; use crate::Progress; use git_odb::FindExt; @@ -27,14 +27,6 @@ mod error { } pub use error::Error; -/// The outcome of receiving a pack via [`Prepare::receive()`]. -#[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)] -#[cfg_attr(feature = "serde1", derive(serde::Serialize, serde::Deserialize))] -pub struct Outcome { - /// Information collected while writing the pack and its index. - pub write_pack_bundle: git_pack::bundle::write::Outcome, -} - /// pub mod negotiate; @@ -76,7 +68,7 @@ where /// /// "fetch.negotiationAlgorithm" describes algorithms `git` uses currently, with the default being `consecutive` and `skipping` being /// experimented with. We currently implement something we could call 'naive' which works for now. - pub fn receive(mut self, should_interrupt: &AtomicBool) -> Result, Error> { + pub fn receive(mut self, should_interrupt: &AtomicBool) -> Result, Error> { let mut con = self.con.take().expect("receive() can only be called once"); let handshake = &self.ref_map.handshake; @@ -106,7 +98,10 @@ where ) { Ok(_) if arguments.is_empty() => { git_protocol::fetch::indicate_end_of_interaction(&mut con.transport).ok(); - return Ok(None); + return Ok(Outcome { + ref_map: self.take_ref_map(), + status: Status::NoChange, + }); } Ok(is_done) => is_done, Err(err) => { @@ -155,7 +150,24 @@ where } // TODO: apply refs - Ok(Some(Outcome { write_pack_bundle })) + Ok(Outcome { + ref_map: self.take_ref_map(), + status: Status::Change { write_pack_bundle }, + }) + } + + fn take_ref_map(&mut self) -> RefMap<'remote> { + let ref_map = RefMap { + mappings: Default::default(), + fixes: Default::default(), + remote_refs: Default::default(), + handshake: git_protocol::fetch::handshake::Outcome { + server_protocol_version: Default::default(), + refs: None, + capabilities: git_protocol::transport::client::Capabilities::default(), + }, + }; + std::mem::replace(&mut self.ref_map, ref_map) } } diff --git a/git-repository/src/remote/mod.rs b/git-repository/src/remote/mod.rs index 4be4fa4dea0..d7db49e17dd 100644 --- a/git-repository/src/remote/mod.rs +++ b/git-repository/src/remote/mod.rs @@ -31,7 +31,7 @@ pub mod fetch { use crate::bstr::BString; /// Information about the relationship between our refspecs, and remote references with their local counterparts. - #[derive(Debug, Clone)] + #[derive(Default, Debug, Clone)] pub struct RefMap<'spec> { /// A mapping between a remote reference and a local tracking branch. pub mappings: Vec, @@ -75,6 +75,27 @@ pub mod fetch { pub spec_index: usize, } + /// The status of the repository after the fetch operation + #[derive(Debug, Clone)] + pub enum Status { + /// Nothing changed as the remote didn't have anything new. + NoChange, + /// There was at least one tip with a new object which we received. + Change { + /// Information collected while writing the pack and its index. + write_pack_bundle: git_pack::bundle::write::Outcome, + }, + } + + /// The outcome of receiving a pack via [`Prepare::receive()`]. + #[derive(Debug, Clone)] + pub struct Outcome<'spec> { + /// The result of the initial mapping of references, the prerequisite for any fetch. + pub ref_map: RefMap<'spec>, + /// The status of the operation to indicate what happened. + pub status: Status, + } + #[cfg(feature = "blocking-network-client")] pub use super::connection::fetch::Prepare; } diff --git a/git-repository/tests/remote/fetch.rs b/git-repository/tests/remote/fetch.rs index 1483854158f..102ff48f0d6 100644 --- a/git-repository/tests/remote/fetch.rs +++ b/git-repository/tests/remote/fetch.rs @@ -36,7 +36,7 @@ mod blocking_io { .connect(Fetch, progress::Discard)? .prepare_fetch(Default::default())? .receive(&AtomicBool::default())?; - assert_eq!(outcome, None, "there is nothing to do right after a clone."); + assert!(matches!(outcome.status, git::remote::fetch::Status::NoChange)); } Ok(()) } From b46347fd3d50886eeca500e31e1e12b354711309 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 26 Sep 2022 08:18:26 +0800 Subject: [PATCH 059/113] rename!: `index::write::Outcome::index_kind` -> `::index_version`. (#450) --- git-pack/src/index/write/mod.rs | 4 ++-- git-pack/tests/pack/bundle.rs | 2 +- git-pack/tests/pack/index.rs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/git-pack/src/index/write/mod.rs b/git-pack/src/index/write/mod.rs index 5b4f2f6a507..9aaeea3a715 100644 --- a/git-pack/src/index/write/mod.rs +++ b/git-pack/src/index/write/mod.rs @@ -18,7 +18,7 @@ pub(crate) struct TreeEntry { #[cfg_attr(feature = "serde1", derive(serde::Serialize, serde::Deserialize))] pub struct Outcome { /// The version of the verified index - pub index_kind: crate::index::Version, + pub index_version: crate::index::Version, /// The verified checksum of the verified index pub index_hash: git_hash::ObjectId, @@ -207,7 +207,7 @@ impl crate::index::File { progress::MessageLevel::Success, ); Ok(Outcome { - index_kind: kind, + index_version: kind, index_hash, data_hash: pack_hash, num_objects, diff --git a/git-pack/tests/pack/bundle.rs b/git-pack/tests/pack/bundle.rs index 04fa4bc136f..dc6faf0489c 100644 --- a/git-pack/tests/pack/bundle.rs +++ b/git-pack/tests/pack/bundle.rs @@ -90,7 +90,7 @@ mod write_to_directory { fn expected_outcome() -> Result> { Ok(pack::bundle::write::Outcome { index: pack::index::write::Outcome { - index_kind: pack::index::Version::V2, + index_version: pack::index::Version::V2, index_hash: git_hash::ObjectId::from_hex(b"544a7204a55f6e9cacccf8f6e191ea8f83575de3")?, data_hash: git_hash::ObjectId::from_hex(b"0f3ea84cd1bba10c2a03d736a460635082833e59")?, num_objects: 42, diff --git a/git-pack/tests/pack/index.rs b/git-pack/tests/pack/index.rs index 56af3c3378d..4e3930d3674 100644 --- a/git-pack/tests/pack/index.rs +++ b/git-pack/tests/pack/index.rs @@ -205,7 +205,7 @@ mod version { outcome.num_objects, num_objects, "it wrote the entire iterator worth of entries" ); - assert_eq!(outcome.index_kind, desired_kind); + assert_eq!(outcome.index_version, desired_kind); assert_eq!( outcome.index_hash, git_hash::ObjectId::from(&expected[end_of_pack_hash..end_of_index_hash]) From 474156f4944763c6e53d01cffb2534791d517598 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 26 Sep 2022 08:20:10 +0800 Subject: [PATCH 060/113] adapt to changes in `git-pack` (#450) --- gitoxide-core/src/pack/receive.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gitoxide-core/src/pack/receive.rs b/gitoxide-core/src/pack/receive.rs index e42634ddfac..93217d2c923 100644 --- a/gitoxide-core/src/pack/receive.rs +++ b/gitoxide-core/src/pack/receive.rs @@ -246,7 +246,7 @@ pub use self::async_io::receive; #[cfg_attr(feature = "serde1", derive(serde::Serialize, serde::Deserialize))] pub struct JsonBundleWriteOutcome { - pub index_kind: pack::index::Version, + pub index_version: pack::index::Version, pub index_hash: String, pub data_hash: String, @@ -256,7 +256,7 @@ pub struct JsonBundleWriteOutcome { impl From for JsonBundleWriteOutcome { fn from(v: pack::index::write::Outcome) -> Self { JsonBundleWriteOutcome { - index_kind: v.index_kind, + index_version: v.index_version, num_objects: v.num_objects, data_hash: v.data_hash.to_string(), index_hash: v.index_hash.to_string(), From 2962dc28c8e93ac81bde70dacfc3081aa697676f Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 26 Sep 2022 08:20:19 +0800 Subject: [PATCH 061/113] A first sketch of validating a fetch. (#450) However, the test generation itself must be optimized and probably move into its own file. --- .../src/remote/connection/fetch/negotiate.rs | 11 ++-- .../tests/fixtures/make_remote_repos.sh | 13 +++++ git-repository/tests/remote/fetch.rs | 51 +++++++++++++++---- git-repository/tests/remote/mod.rs | 14 ++++- 4 files changed, 74 insertions(+), 15 deletions(-) diff --git a/git-repository/src/remote/connection/fetch/negotiate.rs b/git-repository/src/remote/connection/fetch/negotiate.rs index 7aad05a8e1d..cbf7216ecb0 100644 --- a/git-repository/src/remote/connection/fetch/negotiate.rs +++ b/git-repository/src/remote/connection/fetch/negotiate.rs @@ -25,15 +25,20 @@ pub(crate) fn one_round( Algorithm::Naive => { assert_eq!(round, 1, "Naive always finishes after the first round, and claims."); for mapping in &ref_map.mappings { - if let Some(have_id) = mapping.local.as_ref().and_then(|name| { + let have_id = mapping.local.as_ref().and_then(|name| { repo.find_reference(name) .ok() .and_then(|r| r.target().try_id().map(ToOwned::to_owned)) - }) { - if mapping.remote.as_id() != have_id { + }); + match have_id { + Some(have_id) if mapping.remote.as_id() != have_id => { arguments.want(mapping.remote.as_id()); arguments.have(have_id); } + Some(_) => {} + None => { + arguments.want(mapping.remote.as_id()); + } } } Ok(true) diff --git a/git-repository/tests/fixtures/make_remote_repos.sh b/git-repository/tests/fixtures/make_remote_repos.sh index 2f05bac7822..b981d900343 100644 --- a/git-repository/tests/fixtures/make_remote_repos.sh +++ b/git-repository/tests/fixtures/make_remote_repos.sh @@ -97,6 +97,19 @@ git clone --shared base clone git remote add myself . ) +git clone --shared base clone-as-base-with-changes +(cd clone-as-base-with-changes + touch new-file + git add new-file + git commit -m "add new-file" + git tag -m "new-file introduction" v1.0 +) + +git clone --shared base two-origins +(cd two-origins + git remote add changes-on-top-of-origin $PWD/../clone-as-base-with-changes +) + git clone --shared base push-default (cd push-default diff --git a/git-repository/tests/remote/fetch.rs b/git-repository/tests/remote/fetch.rs index 102ff48f0d6..846ba745461 100644 --- a/git-repository/tests/remote/fetch.rs +++ b/git-repository/tests/remote/fetch.rs @@ -2,7 +2,9 @@ mod blocking_io { use git_features::progress; use git_repository as git; + use git_repository::remote::fetch; use git_repository::remote::Direction::Fetch; + use git_testtools::hex_to_id; use std::sync::atomic::AtomicBool; use crate::remote; @@ -14,7 +16,7 @@ mod blocking_io { Some(git::protocol::transport::Protocol::V2), Some(git::protocol::transport::Protocol::V1), ] { - let mut repo = remote::repo("clone"); + let (mut repo, _tmp) = remote::repo_rw("two-origins"); if let Some(version) = version { repo.config_snapshot_mut().set_raw_value( "protocol", @@ -24,19 +26,46 @@ mod blocking_io { )?; } - let remote = repo.find_remote("origin")?; + // No updates { - remote + let remote = repo.find_remote("origin")?; + { + remote + .connect(Fetch, progress::Discard)? + .prepare_fetch(Default::default())?; + // early drops are fine and won't block. + } + let outcome = remote .connect(Fetch, progress::Discard)? - .prepare_fetch(Default::default())?; - // early drops are fine and won't block. + .prepare_fetch(Default::default())? + .receive(&AtomicBool::default())?; + assert!(matches!(outcome.status, git::remote::fetch::Status::NoChange)); + } + + // Some updates to be fetched + { + let remote = repo.find_remote("changes-on-top-of-origin")?; + let outcome: git::remote::fetch::Outcome = remote + .connect(Fetch, progress::Discard)? + .prepare_fetch(Default::default())? + .receive(&AtomicBool::default())?; + match outcome.status { + fetch::Status::Change { write_pack_bundle } => { + assert_eq!(write_pack_bundle.pack_kind, git::odb::pack::data::Version::V2); + assert_eq!(write_pack_bundle.object_hash, repo.object_hash()); + assert_eq!(write_pack_bundle.index.num_objects, 33); // TODO: should just be 4! but in naive mode it's what happens currently. + assert_eq!( + write_pack_bundle.index.index_version, + git::odb::pack::index::Version::V2 + ); + assert_eq!( + write_pack_bundle.index.index_hash, + hex_to_id("1e396d0e2ab415556b240dc6251c65c71b568caa") + ); + } + fetch::Status::NoChange => unreachable!("we firmly expect changes here"), + } } - // TODO: make sure there is actually something to do to run into other issues - let outcome = remote - .connect(Fetch, progress::Discard)? - .prepare_fetch(Default::default())? - .receive(&AtomicBool::default())?; - assert!(matches!(outcome.status, git::remote::fetch::Status::NoChange)); } Ok(()) } diff --git a/git-repository/tests/remote/mod.rs b/git-repository/tests/remote/mod.rs index ea4b5c03e75..ac3859adf44 100644 --- a/git-repository/tests/remote/mod.rs +++ b/git-repository/tests/remote/mod.rs @@ -1,7 +1,8 @@ use std::{borrow::Cow, path::PathBuf}; +use tempfile::TempDir; use git_repository as git; -use git_testtools::scripted_fixture_repo_read_only; +use git_testtools::{scripted_fixture_repo_read_only, scripted_fixture_repo_writable_with_args}; pub(crate) fn repo_path(name: &str) -> PathBuf { let dir = scripted_fixture_repo_read_only("make_remote_repos.sh").unwrap(); @@ -12,6 +13,17 @@ pub(crate) fn repo(name: &str) -> git::Repository { git::open_opts(repo_path(name), git::open::Options::isolated()).unwrap() } +pub(crate) fn repo_rw(name: &str) -> (git::Repository, TempDir) { + let dir = scripted_fixture_repo_writable_with_args( + "make_remote_repos.sh", + &[] as &[String], + git_testtools::Creation::ExecuteScript, + ) + .unwrap(); + let repo = git::open_opts(dir.path().join(name), git::open::Options::isolated()).unwrap(); + (repo, dir) +} + pub(crate) fn cow_str(s: &str) -> Cow { Cow::Borrowed(s) } From 8499c3e8afba0767fe9f0ca0e3016fe9f84951e5 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 26 Sep 2022 08:22:42 +0800 Subject: [PATCH 062/113] thanks clippy --- git-repository/tests/remote/mod.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/git-repository/tests/remote/mod.rs b/git-repository/tests/remote/mod.rs index ac3859adf44..bbeb581e767 100644 --- a/git-repository/tests/remote/mod.rs +++ b/git-repository/tests/remote/mod.rs @@ -1,8 +1,7 @@ use std::{borrow::Cow, path::PathBuf}; -use tempfile::TempDir; use git_repository as git; -use git_testtools::{scripted_fixture_repo_read_only, scripted_fixture_repo_writable_with_args}; +use git_testtools::scripted_fixture_repo_read_only; pub(crate) fn repo_path(name: &str) -> PathBuf { let dir = scripted_fixture_repo_read_only("make_remote_repos.sh").unwrap(); @@ -13,8 +12,10 @@ pub(crate) fn repo(name: &str) -> git::Repository { git::open_opts(repo_path(name), git::open::Options::isolated()).unwrap() } -pub(crate) fn repo_rw(name: &str) -> (git::Repository, TempDir) { - let dir = scripted_fixture_repo_writable_with_args( +#[cfg(feature = "blocking-network-client")] +// TODO: move this to where it's used (fetch) +pub(crate) fn repo_rw(name: &str) -> (git::Repository, git_testtools::tempfile::TempDir) { + let dir = git_testtools::scripted_fixture_repo_writable_with_args( "make_remote_repos.sh", &[] as &[String], git_testtools::Creation::ExecuteScript, From f8fe6e44b5e041fb2275fce490695452e072ee17 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 26 Sep 2022 08:39:46 +0800 Subject: [PATCH 063/113] fix journey tests (#450) --- .../no-repo/pack/index/create/no-output-dir-as-json-success | 2 +- .../pack/index/create/output-dir-restore-as-json-success | 2 +- .../plumbing/no-repo/pack/receive/file-v-any-no-output-json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/snapshots/plumbing/no-repo/pack/index/create/no-output-dir-as-json-success b/tests/snapshots/plumbing/no-repo/pack/index/create/no-output-dir-as-json-success index fef8d1982c5..d0a2c67ba71 100644 --- a/tests/snapshots/plumbing/no-repo/pack/index/create/no-output-dir-as-json-success +++ b/tests/snapshots/plumbing/no-repo/pack/index/create/no-output-dir-as-json-success @@ -1,6 +1,6 @@ { "index": { - "index_kind": "V2", + "index_version": "V2", "index_hash": { "Sha1": [ 86, diff --git a/tests/snapshots/plumbing/no-repo/pack/index/create/output-dir-restore-as-json-success b/tests/snapshots/plumbing/no-repo/pack/index/create/output-dir-restore-as-json-success index e3254bf57d6..9fa80d16c06 100644 --- a/tests/snapshots/plumbing/no-repo/pack/index/create/output-dir-restore-as-json-success +++ b/tests/snapshots/plumbing/no-repo/pack/index/create/output-dir-restore-as-json-success @@ -1,6 +1,6 @@ { "index": { - "index_kind": "V2", + "index_version": "V2", "index_hash": { "Sha1": [ 44, diff --git a/tests/snapshots/plumbing/no-repo/pack/receive/file-v-any-no-output-json b/tests/snapshots/plumbing/no-repo/pack/receive/file-v-any-no-output-json index 590a10265e2..ab2bb31f86b 100644 --- a/tests/snapshots/plumbing/no-repo/pack/receive/file-v-any-no-output-json +++ b/tests/snapshots/plumbing/no-repo/pack/receive/file-v-any-no-output-json @@ -1,6 +1,6 @@ { "index": { - "index_kind": "V2", + "index_version": "V2", "index_hash": "c787de2aafb897417ca8167baeb146eabd18bc5f", "data_hash": "346574b7331dc3a1724da218d622c6e1b6c66a57", "num_objects": 9 From ce1a373c80e076c148112532990b781044b7aeb8 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 26 Sep 2022 12:23:40 +0800 Subject: [PATCH 064/113] speed up fetch tests by giving them their own repo-fixture (#450) --- .../fixtures/generated-archives/.gitignore | 1 + .../tests/fixtures/make_fetch_repos.sh | 106 ++++++++++++++++++ .../tests/fixtures/make_remote_repos.sh | 13 --- git-repository/tests/remote/fetch.rs | 13 ++- git-repository/tests/remote/mod.rs | 13 --- 5 files changed, 118 insertions(+), 28 deletions(-) create mode 100644 git-repository/tests/fixtures/make_fetch_repos.sh diff --git a/git-repository/tests/fixtures/generated-archives/.gitignore b/git-repository/tests/fixtures/generated-archives/.gitignore index 9250e9825f6..9f4f7db5774 100644 --- a/git-repository/tests/fixtures/generated-archives/.gitignore +++ b/git-repository/tests/fixtures/generated-archives/.gitignore @@ -1,3 +1,4 @@ /make_worktree_repo.tar.xz /make_remote_repos.tar.xz +/make_fetch_repos.tar.xz /make_core_worktree_repo.tar.xz diff --git a/git-repository/tests/fixtures/make_fetch_repos.sh b/git-repository/tests/fixtures/make_fetch_repos.sh new file mode 100644 index 00000000000..34df81800c8 --- /dev/null +++ b/git-repository/tests/fixtures/make_fetch_repos.sh @@ -0,0 +1,106 @@ +set -eu -o pipefail + +function tick () { + if test -z "${tick+set}" + then + tick=1112911993 + else + tick=$(($tick + 60)) + fi + GIT_COMMITTER_DATE="$tick -0700" + GIT_AUTHOR_DATE="$tick -0700" + export GIT_COMMITTER_DATE GIT_AUTHOR_DATE +} + +GIT_AUTHOR_EMAIL=author@example.com +GIT_AUTHOR_NAME='A U Thor' +GIT_AUTHOR_DATE='1112354055 +0200' +TEST_COMMITTER_LOCALNAME=committer +TEST_COMMITTER_DOMAIN=example.com +GIT_COMMITTER_EMAIL=committer@example.com +GIT_COMMITTER_NAME='C O Mitter' +GIT_COMMITTER_DATE='1112354055 +0200' + +# runup to the correct count for ambigous commits +tick; tick; tick; tick; tick + +git init base +( + cd base + tick + + echo g > file + git add file && git commit -m $'G\n\n initial message' + git branch g + + tick + git checkout --orphan=h + echo h > file + git add file && git commit -m H + + tick + git checkout main + git merge h --allow-unrelated-histories || : + { echo g && echo h && echo d; } > file + git add file + git commit -m D + git branch d + + tick + git checkout --orphan=i + echo i > file + git add file && git commit -m I + git tag -m I-tag i-tag + + tick + git checkout --orphan=j + echo j > file + git add file && git commit -m J + + tick + git checkout i + git merge j --allow-unrelated-histories || : + { echo i && echo j && echo f; } > file + git add file + git commit -m F + git branch f + + tick + git checkout --orphan=e + echo e > file + git add file && git commit -m E + + tick + git checkout main + git merge e i --allow-unrelated-histories || : + { echo g && echo h && echo i && echo j && echo d && echo e && echo f && echo b; } > file + git add file && git commit -m B + git tag -m b-tag b-tag && git branch b + + tick + git checkout i + echo c >> file + git add file && git commit -m $'C\n\n message recent' + git branch c + git reset --hard i-tag + + tick + git checkout main + git merge c || : + { echo g && echo h && echo i && echo j && echo d && echo e && echo f && echo b && echo c && echo a; } > file + git add file && git commit -m A + git branch a +) + +git clone --shared base clone-as-base-with-changes +(cd clone-as-base-with-changes + touch new-file + git add new-file + git commit -m "add new-file" + git tag -m "new-file introduction" v1.0 +) + +git clone --shared base two-origins +(cd two-origins + git remote add changes-on-top-of-origin "$PWD/../clone-as-base-with-changes" +) diff --git a/git-repository/tests/fixtures/make_remote_repos.sh b/git-repository/tests/fixtures/make_remote_repos.sh index b981d900343..2f05bac7822 100644 --- a/git-repository/tests/fixtures/make_remote_repos.sh +++ b/git-repository/tests/fixtures/make_remote_repos.sh @@ -97,19 +97,6 @@ git clone --shared base clone git remote add myself . ) -git clone --shared base clone-as-base-with-changes -(cd clone-as-base-with-changes - touch new-file - git add new-file - git commit -m "add new-file" - git tag -m "new-file introduction" v1.0 -) - -git clone --shared base two-origins -(cd two-origins - git remote add changes-on-top-of-origin $PWD/../clone-as-base-with-changes -) - git clone --shared base push-default (cd push-default diff --git a/git-repository/tests/remote/fetch.rs b/git-repository/tests/remote/fetch.rs index 846ba745461..08a11ffcbda 100644 --- a/git-repository/tests/remote/fetch.rs +++ b/git-repository/tests/remote/fetch.rs @@ -7,7 +7,16 @@ mod blocking_io { use git_testtools::hex_to_id; use std::sync::atomic::AtomicBool; - use crate::remote; + pub(crate) fn repo_rw(name: &str) -> (git::Repository, git_testtools::tempfile::TempDir) { + let dir = git_testtools::scripted_fixture_repo_writable_with_args( + "make_fetch_repos.sh", + &[] as &[String], + git_testtools::Creation::ExecuteScript, + ) + .unwrap(); + let repo = git::open_opts(dir.path().join(name), git::open::Options::isolated()).unwrap(); + (repo, dir) + } #[test] fn fetch_pack() -> crate::Result { @@ -16,7 +25,7 @@ mod blocking_io { Some(git::protocol::transport::Protocol::V2), Some(git::protocol::transport::Protocol::V1), ] { - let (mut repo, _tmp) = remote::repo_rw("two-origins"); + let (mut repo, _tmp) = repo_rw("two-origins"); if let Some(version) = version { repo.config_snapshot_mut().set_raw_value( "protocol", diff --git a/git-repository/tests/remote/mod.rs b/git-repository/tests/remote/mod.rs index bbeb581e767..ea4b5c03e75 100644 --- a/git-repository/tests/remote/mod.rs +++ b/git-repository/tests/remote/mod.rs @@ -12,19 +12,6 @@ pub(crate) fn repo(name: &str) -> git::Repository { git::open_opts(repo_path(name), git::open::Options::isolated()).unwrap() } -#[cfg(feature = "blocking-network-client")] -// TODO: move this to where it's used (fetch) -pub(crate) fn repo_rw(name: &str) -> (git::Repository, git_testtools::tempfile::TempDir) { - let dir = git_testtools::scripted_fixture_repo_writable_with_args( - "make_remote_repos.sh", - &[] as &[String], - git_testtools::Creation::ExecuteScript, - ) - .unwrap(); - let repo = git::open_opts(dir.path().join(name), git::open::Options::isolated()).unwrap(); - (repo, dir) -} - pub(crate) fn cow_str(s: &str) -> Cow { Cow::Borrowed(s) } From d5c1f9280e8a20f8ce8c087bde04f4098cafe993 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 26 Sep 2022 13:45:21 +0800 Subject: [PATCH 065/113] =?UTF-8?q?try=20to=20make=20naive=20negotiation?= =?UTF-8?q?=20better,=20but=E2=80=A6=20(#450)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …turns out that trying to analyse remotes to know what to send doesn't help as it's hard to validate we send the right remote tracking branches. After all, they should belong to the smae repo. --- .../src/remote/connection/fetch/mod.rs | 2 +- .../src/remote/connection/fetch/negotiate.rs | 57 ++++++++++++++++++- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/git-repository/src/remote/connection/fetch/mod.rs b/git-repository/src/remote/connection/fetch/mod.rs index f81dfa4d0db..873dbfc56d1 100644 --- a/git-repository/src/remote/connection/fetch/mod.rs +++ b/git-repository/src/remote/connection/fetch/mod.rs @@ -91,7 +91,7 @@ where let is_done = match negotiate::one_round( negotiate::Algorithm::Naive, round, - repo, + con.remote, &self.ref_map, &mut arguments, previous_response.as_ref(), diff --git a/git-repository/src/remote/connection/fetch/negotiate.rs b/git-repository/src/remote/connection/fetch/negotiate.rs index cbf7216ecb0..2a7d1b7d427 100644 --- a/git-repository/src/remote/connection/fetch/negotiate.rs +++ b/git-repository/src/remote/connection/fetch/negotiate.rs @@ -1,3 +1,5 @@ +use crate::bstr::{BStr, ByteSlice}; + #[derive(Copy, Clone)] pub(crate) enum Algorithm { /// Our very own implementation that probably should be replaced by one of the known algorithms soon. @@ -16,14 +18,16 @@ pub enum Error { pub(crate) fn one_round( algo: Algorithm, round: usize, - repo: &crate::Repository, + remote: &crate::Remote<'_>, ref_map: &crate::remote::fetch::RefMap<'_>, arguments: &mut git_protocol::fetch::Arguments, _previous_response: Option<&git_protocol::fetch::Response>, ) -> Result { + let repo = remote.repo; match algo { Algorithm::Naive => { assert_eq!(round, 1, "Naive always finishes after the first round, and claims."); + let mut has_missing_tracking_branch = false; for mapping in &ref_map.mappings { let have_id = mapping.local.as_ref().and_then(|name| { repo.find_reference(name) @@ -38,6 +42,25 @@ pub(crate) fn one_round( Some(_) => {} None => { arguments.want(mapping.remote.as_id()); + has_missing_tracking_branch = true; + } + } + } + + if has_missing_tracking_branch { + let our_url = remote + .url(crate::remote::Direction::Fetch) + .expect("url present or we wouldn't be here"); + let our_repo_name = strip_git_suffix(our_url.path.as_ref()); + dbg!(our_repo_name); + for other_remote in repo.remote_names().iter().filter_map(|name| match remote.name() { + Some(our_name) if our_name == *name => None, + Some(_) | None => repo.find_remote(name).ok(), + }) { + if let Some(other_url) = other_remote.url(crate::remote::Direction::Fetch) { + if strip_git_suffix(other_url.path.as_ref()) == our_repo_name { + dbg!(&other_url, &our_url); + } } } } @@ -45,3 +68,35 @@ pub(crate) fn one_round( } } } + +fn strip_git_suffix(repo_path: &BStr) -> &BStr { + let repo_path = repo_path.strip_suffix(b"/.git").unwrap_or(repo_path); + let repo_path = repo_path + .rfind_byte(b'/') + .map(|slash| &repo_path[slash + 1..]) + .unwrap_or(repo_path); + repo_path + .strip_suffix(b".git") + .map(Into::into) + .unwrap_or(repo_path.into()) +} + +#[cfg(test)] +mod strip_git_suffix_tests { + use super::strip_git_suffix; + + #[test] + fn dot_git_dir() { + assert_eq!(strip_git_suffix("a/repo/.git".into()), "repo"); + } + + #[test] + fn dot_git_suffix() { + assert_eq!(strip_git_suffix("/a/repo.git".into()), "repo"); + } + + #[test] + fn no_git_suffix() { + assert_eq!(strip_git_suffix("a/b/repo".into()), "repo"); + } +} From 038779420edddb651c3463e4679778ceabf902b8 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 26 Sep 2022 13:51:23 +0800 Subject: [PATCH 066/113] improve naieve algorithm to be a bit better in our test-case (#450) --- .../src/remote/connection/fetch/negotiate.rs | 50 ++----------------- git-repository/tests/remote/fetch.rs | 4 +- 2 files changed, 5 insertions(+), 49 deletions(-) diff --git a/git-repository/src/remote/connection/fetch/negotiate.rs b/git-repository/src/remote/connection/fetch/negotiate.rs index 2a7d1b7d427..073710f42d8 100644 --- a/git-repository/src/remote/connection/fetch/negotiate.rs +++ b/git-repository/src/remote/connection/fetch/negotiate.rs @@ -1,5 +1,3 @@ -use crate::bstr::{BStr, ByteSlice}; - #[derive(Copy, Clone)] pub(crate) enum Algorithm { /// Our very own implementation that probably should be replaced by one of the known algorithms soon. @@ -48,19 +46,9 @@ pub(crate) fn one_round( } if has_missing_tracking_branch { - let our_url = remote - .url(crate::remote::Direction::Fetch) - .expect("url present or we wouldn't be here"); - let our_repo_name = strip_git_suffix(our_url.path.as_ref()); - dbg!(our_repo_name); - for other_remote in repo.remote_names().iter().filter_map(|name| match remote.name() { - Some(our_name) if our_name == *name => None, - Some(_) | None => repo.find_remote(name).ok(), - }) { - if let Some(other_url) = other_remote.url(crate::remote::Direction::Fetch) { - if strip_git_suffix(other_url.path.as_ref()) == our_repo_name { - dbg!(&other_url, &our_url); - } + if let Ok(Some(r)) = repo.head_ref() { + if let Some(id) = r.target().try_id() { + arguments.have(id); } } } @@ -68,35 +56,3 @@ pub(crate) fn one_round( } } } - -fn strip_git_suffix(repo_path: &BStr) -> &BStr { - let repo_path = repo_path.strip_suffix(b"/.git").unwrap_or(repo_path); - let repo_path = repo_path - .rfind_byte(b'/') - .map(|slash| &repo_path[slash + 1..]) - .unwrap_or(repo_path); - repo_path - .strip_suffix(b".git") - .map(Into::into) - .unwrap_or(repo_path.into()) -} - -#[cfg(test)] -mod strip_git_suffix_tests { - use super::strip_git_suffix; - - #[test] - fn dot_git_dir() { - assert_eq!(strip_git_suffix("a/repo/.git".into()), "repo"); - } - - #[test] - fn dot_git_suffix() { - assert_eq!(strip_git_suffix("/a/repo.git".into()), "repo"); - } - - #[test] - fn no_git_suffix() { - assert_eq!(strip_git_suffix("a/b/repo".into()), "repo"); - } -} diff --git a/git-repository/tests/remote/fetch.rs b/git-repository/tests/remote/fetch.rs index 08a11ffcbda..0cc91b8cd74 100644 --- a/git-repository/tests/remote/fetch.rs +++ b/git-repository/tests/remote/fetch.rs @@ -62,14 +62,14 @@ mod blocking_io { fetch::Status::Change { write_pack_bundle } => { assert_eq!(write_pack_bundle.pack_kind, git::odb::pack::data::Version::V2); assert_eq!(write_pack_bundle.object_hash, repo.object_hash()); - assert_eq!(write_pack_bundle.index.num_objects, 33); // TODO: should just be 4! but in naive mode it's what happens currently. + assert_eq!(write_pack_bundle.index.num_objects, 3, "this value is 4 when git does it with 'consecutive' negotiation style, but could be 33 if completely naive."); assert_eq!( write_pack_bundle.index.index_version, git::odb::pack::index::Version::V2 ); assert_eq!( write_pack_bundle.index.index_hash, - hex_to_id("1e396d0e2ab415556b240dc6251c65c71b568caa") + hex_to_id("5e0c69c18bf1835edaa103622dc8637fd87ea2f3") ); } fetch::Status::NoChange => unreachable!("we firmly expect changes here"), From 2ec8175a55a9cd02408cab45d84da2823c44dec4 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 26 Sep 2022 14:01:11 +0800 Subject: [PATCH 067/113] greatly improved performance for write-test. (#450) It clones instead of re-creating the base repo from scratch. --- .../tests/fixtures/make_fetch_repos.sh | 94 +------------------ git-repository/tests/remote/fetch.rs | 9 +- 2 files changed, 9 insertions(+), 94 deletions(-) diff --git a/git-repository/tests/fixtures/make_fetch_repos.sh b/git-repository/tests/fixtures/make_fetch_repos.sh index 34df81800c8..cecc027d442 100644 --- a/git-repository/tests/fixtures/make_fetch_repos.sh +++ b/git-repository/tests/fixtures/make_fetch_repos.sh @@ -1,96 +1,6 @@ set -eu -o pipefail -function tick () { - if test -z "${tick+set}" - then - tick=1112911993 - else - tick=$(($tick + 60)) - fi - GIT_COMMITTER_DATE="$tick -0700" - GIT_AUTHOR_DATE="$tick -0700" - export GIT_COMMITTER_DATE GIT_AUTHOR_DATE -} - -GIT_AUTHOR_EMAIL=author@example.com -GIT_AUTHOR_NAME='A U Thor' -GIT_AUTHOR_DATE='1112354055 +0200' -TEST_COMMITTER_LOCALNAME=committer -TEST_COMMITTER_DOMAIN=example.com -GIT_COMMITTER_EMAIL=committer@example.com -GIT_COMMITTER_NAME='C O Mitter' -GIT_COMMITTER_DATE='1112354055 +0200' - -# runup to the correct count for ambigous commits -tick; tick; tick; tick; tick - -git init base -( - cd base - tick - - echo g > file - git add file && git commit -m $'G\n\n initial message' - git branch g - - tick - git checkout --orphan=h - echo h > file - git add file && git commit -m H - - tick - git checkout main - git merge h --allow-unrelated-histories || : - { echo g && echo h && echo d; } > file - git add file - git commit -m D - git branch d - - tick - git checkout --orphan=i - echo i > file - git add file && git commit -m I - git tag -m I-tag i-tag - - tick - git checkout --orphan=j - echo j > file - git add file && git commit -m J - - tick - git checkout i - git merge j --allow-unrelated-histories || : - { echo i && echo j && echo f; } > file - git add file - git commit -m F - git branch f - - tick - git checkout --orphan=e - echo e > file - git add file && git commit -m E - - tick - git checkout main - git merge e i --allow-unrelated-histories || : - { echo g && echo h && echo i && echo j && echo d && echo e && echo f && echo b; } > file - git add file && git commit -m B - git tag -m b-tag b-tag && git branch b - - tick - git checkout i - echo c >> file - git add file && git commit -m $'C\n\n message recent' - git branch c - git reset --hard i-tag - - tick - git checkout main - git merge c || : - { echo g && echo h && echo i && echo j && echo d && echo e && echo f && echo b && echo c && echo a; } > file - git add file && git commit -m A - git branch a -) +git clone --bare "${1:?First argument is the complex base repo from make_remote_repos.sh/base}" base git clone --shared base clone-as-base-with-changes (cd clone-as-base-with-changes @@ -100,7 +10,7 @@ git clone --shared base clone-as-base-with-changes git tag -m "new-file introduction" v1.0 ) -git clone --shared base two-origins +git clone --bare --shared base two-origins (cd two-origins git remote add changes-on-top-of-origin "$PWD/../clone-as-base-with-changes" ) diff --git a/git-repository/tests/remote/fetch.rs b/git-repository/tests/remote/fetch.rs index 0cc91b8cd74..6f675699f76 100644 --- a/git-repository/tests/remote/fetch.rs +++ b/git-repository/tests/remote/fetch.rs @@ -1,5 +1,6 @@ #[cfg(feature = "blocking-network-client")] mod blocking_io { + use crate::remote; use git_features::progress; use git_repository as git; use git_repository::remote::fetch; @@ -10,7 +11,9 @@ mod blocking_io { pub(crate) fn repo_rw(name: &str) -> (git::Repository, git_testtools::tempfile::TempDir) { let dir = git_testtools::scripted_fixture_repo_writable_with_args( "make_fetch_repos.sh", - &[] as &[String], + [git::path::realpath(remote::repo_path("base")) + .unwrap() + .to_string_lossy()], git_testtools::Creation::ExecuteScript, ) .unwrap(); @@ -69,8 +72,10 @@ mod blocking_io { ); assert_eq!( write_pack_bundle.index.index_hash, - hex_to_id("5e0c69c18bf1835edaa103622dc8637fd87ea2f3") + hex_to_id("c75114f60ab2c9389916f3de1082bbaa47491e3b") ); + assert!(write_pack_bundle.data_path.map_or(false, |f| f.is_file())); + assert!(write_pack_bundle.index_path.map_or(false, |f| f.is_file())); } fetch::Status::NoChange => unreachable!("we firmly expect changes here"), } From 96f2fd8d848dd170855721f85ec6386f9391f0a1 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 26 Sep 2022 15:13:58 +0800 Subject: [PATCH 068/113] refactor (#450) --- .../src/remote/connection/fetch/mod.rs | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/git-repository/src/remote/connection/fetch/mod.rs b/git-repository/src/remote/connection/fetch/mod.rs index 873dbfc56d1..537d8eca154 100644 --- a/git-repository/src/remote/connection/fetch/mod.rs +++ b/git-repository/src/remote/connection/fetch/mod.rs @@ -99,7 +99,7 @@ where Ok(_) if arguments.is_empty() => { git_protocol::fetch::indicate_end_of_interaction(&mut con.transport).ok(); return Ok(Outcome { - ref_map: self.take_ref_map(), + ref_map: std::mem::take(&mut self.ref_map), status: Status::NoChange, }); } @@ -151,24 +151,10 @@ where // TODO: apply refs Ok(Outcome { - ref_map: self.take_ref_map(), + ref_map: std::mem::take(&mut self.ref_map), status: Status::Change { write_pack_bundle }, }) } - - fn take_ref_map(&mut self) -> RefMap<'remote> { - let ref_map = RefMap { - mappings: Default::default(), - fixes: Default::default(), - remote_refs: Default::default(), - handshake: git_protocol::fetch::handshake::Outcome { - server_protocol_version: Default::default(), - refs: None, - capabilities: git_protocol::transport::client::Capabilities::default(), - }, - }; - std::mem::replace(&mut self.ref_map, ref_map) - } } fn setup_remote_progress( From 1f2d6095d946f6327e67a7388fd87ab9c74be31d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 26 Sep 2022 16:57:05 +0800 Subject: [PATCH 069/113] lay the ground-works for testing the update of refs (#450) --- .../src/remote/connection/fetch/mod.rs | 16 +++- .../src/remote/connection/fetch/negotiate.rs | 6 +- .../src/remote/connection/fetch/refs.rs | 78 +++++++++++++++++++ git-repository/src/remote/fetch.rs | 72 +++++++++++++++++ git-repository/src/remote/mod.rs | 73 +---------------- git-repository/tests/remote/fetch.rs | 5 +- 6 files changed, 172 insertions(+), 78 deletions(-) create mode 100644 git-repository/src/remote/connection/fetch/refs.rs create mode 100644 git-repository/src/remote/fetch.rs diff --git a/git-repository/src/remote/connection/fetch/mod.rs b/git-repository/src/remote/connection/fetch/mod.rs index 537d8eca154..58a0b47d90c 100644 --- a/git-repository/src/remote/connection/fetch/mod.rs +++ b/git-repository/src/remote/connection/fetch/mod.rs @@ -8,6 +8,7 @@ use std::sync::atomic::AtomicBool; mod error { /// The error returned by [`receive()`](super::Prepare::receive()). #[derive(Debug, thiserror::Error)] + #[allow(missing_docs)] pub enum Error { #[error("{message}{}", desired.map(|n| format!(" (got {})", n)).unwrap_or_default())] Configuration { @@ -23,6 +24,8 @@ mod error { Client(#[from] git_protocol::transport::client::Error), #[error(transparent)] WritePack(#[from] git_pack::bundle::write::Error), + #[error(transparent)] + UpdateRefs(#[from] super::refs::update::Error), } } pub use error::Error; @@ -91,7 +94,7 @@ where let is_done = match negotiate::one_round( negotiate::Algorithm::Naive, round, - con.remote, + repo, &self.ref_map, &mut arguments, previous_response.as_ref(), @@ -133,6 +136,7 @@ where iteration_mode: git_pack::data::input::Mode::Verify, object_hash: con.remote.repo.object_hash(), }; + let remote = con.remote; let write_pack_bundle = git_pack::Bundle::write_to_directory( reader, Some(repo.objects.store_ref().path().join("pack")), @@ -149,10 +153,14 @@ where git_protocol::fetch::indicate_end_of_interaction(&mut con.transport).ok(); } - // TODO: apply refs + let update_refs = refs::update(repo, remote.refspecs(crate::remote::Direction::Fetch), &self.ref_map)?; + Ok(Outcome { ref_map: std::mem::take(&mut self.ref_map), - status: Status::Change { write_pack_bundle }, + status: Status::Change { + write_pack_bundle, + update_refs, + }, }) } } @@ -171,6 +179,8 @@ fn setup_remote_progress( } mod config; +/// +pub mod refs; /// A structure to hold the result of the handshake with the remote and configure the upcoming fetch operation. #[allow(dead_code)] diff --git a/git-repository/src/remote/connection/fetch/negotiate.rs b/git-repository/src/remote/connection/fetch/negotiate.rs index 073710f42d8..f020732b74d 100644 --- a/git-repository/src/remote/connection/fetch/negotiate.rs +++ b/git-repository/src/remote/connection/fetch/negotiate.rs @@ -1,10 +1,13 @@ +/// The way the negotiation is performed #[derive(Copy, Clone)] pub(crate) enum Algorithm { /// Our very own implementation that probably should be replaced by one of the known algorithms soon. Naive, } +/// The error returned during negotiation. #[derive(Debug, thiserror::Error)] +#[allow(missing_docs)] pub enum Error { #[error("We were unable to figure out what objects the server should send after {rounds} round(s)")] NegotiationFailed { rounds: usize }, @@ -16,12 +19,11 @@ pub enum Error { pub(crate) fn one_round( algo: Algorithm, round: usize, - remote: &crate::Remote<'_>, + repo: &crate::Repository, ref_map: &crate::remote::fetch::RefMap<'_>, arguments: &mut git_protocol::fetch::Arguments, _previous_response: Option<&git_protocol::fetch::Response>, ) -> Result { - let repo = remote.repo; match algo { Algorithm::Naive => { assert_eq!(round, 1, "Naive always finishes after the first round, and claims."); diff --git a/git-repository/src/remote/connection/fetch/refs.rs b/git-repository/src/remote/connection/fetch/refs.rs new file mode 100644 index 00000000000..dddd7881c29 --- /dev/null +++ b/git-repository/src/remote/connection/fetch/refs.rs @@ -0,0 +1,78 @@ +use crate::remote::fetch::RefMap; + +/// +pub mod update { + mod error { + /// The error returned when updating refs after a fetch operation. + #[derive(Debug, thiserror::Error)] + #[error("TBD")] + pub struct Error {} + } + use crate::remote::fetch::RefMap; + pub use error::Error; + + /// The outcome of the refs-update operation at the end of a fetch. + #[derive(Debug, Clone)] + pub struct Outcome { + /// All edits that were performed to update local refs. + pub edits: Vec, + /// Each update provides more information about what happened to the corresponding mapping. + /// Use [`iter_mapping_updates()`][Self::iter_mapping_updates()] to recombine the update information with ref-edits and their + /// mapping. + pub updates: Vec, + } + + /// Describe the way a ref was updated + #[derive(Debug, Copy, Clone)] + pub enum Mode { + /// The old ref's commit was an ancestor of the new one, allowing for a fast-forward without a merge. + FastForward, + /// The ref was set to point to the new commit from the remote without taking into consideration its ancestry. + Forced, + /// No change was attempted as the remote ref didn't change compared to the current ref, or because no remote ref was specified + /// in the ref-spec. + NoChangeNeeded, + } + + impl Outcome { + /// Produce an iterator over all information used to produce the this outcome, ref-update by ref-update, using the `ref_map` + /// used when producing the ref update. + pub fn iter_mapping_updates<'a>( + &self, + ref_map: &'a RefMap<'_>, + ) -> impl Iterator< + Item = ( + &super::Update, + &'a crate::remote::fetch::Mapping, + Option<&git_ref::transaction::RefEdit>, + ), + > { + self.updates + .iter() + .zip(ref_map.mappings.iter()) + .map(move |(update, mapping)| (update, mapping, update.edit_index.and_then(|idx| self.edits.get(idx)))) + } + } +} +use git_refspec::RefSpec; + +/// Information about the update of a single reference, corresponding the respective entry in [`RefMap::mapping`]. +#[derive(Debug, Clone, Copy)] +pub struct Update { + /// The way the update was performed. + pub mode: update::Mode, + /// The index to the edit that was created from the corresponding mapping, or `None` if there was no local ref. + pub edit_index: Option, +} + +pub(crate) fn update( + _repo: &crate::Repository, + _remote: &[RefSpec], + _ref_map: &RefMap<'_>, +) -> Result { + // TODO: tests and impl + Ok(update::Outcome { + edits: Default::default(), + updates: Default::default(), + }) +} diff --git a/git-repository/src/remote/fetch.rs b/git-repository/src/remote/fetch.rs new file mode 100644 index 00000000000..682ce2195f6 --- /dev/null +++ b/git-repository/src/remote/fetch.rs @@ -0,0 +1,72 @@ +use crate::bstr::BString; + +/// Information about the relationship between our refspecs, and remote references with their local counterparts. +#[derive(Default, Debug, Clone)] +pub struct RefMap<'spec> { + /// A mapping between a remote reference and a local tracking branch. + pub mappings: Vec, + /// Information about the fixes applied to the `mapping` due to validation and sanitization. + pub fixes: Vec>, + /// All refs advertised by the remote. + pub remote_refs: Vec, + /// Additional information provided by the server as part of the handshake. + /// + /// Note that the `refs` field is always `None` as the refs are placed in `remote_refs`. + pub handshake: git_protocol::fetch::handshake::Outcome, +} + +/// Either an object id that the remote has or the matched remote ref itself. +#[derive(Debug, Clone)] +pub enum Source { + /// An object id, as the matched ref-spec was an object id itself. + ObjectId(git_hash::ObjectId), + /// The remote reference that matched the ref-specs name. + Ref(git_protocol::fetch::Ref), +} + +impl Source { + /// Return either the direct object id we refer to or the direct target that a reference refers to. + pub fn as_id(&self) -> &git_hash::oid { + match self { + Source::ObjectId(id) => id, + Source::Ref(r) => r.unpack().1, + } + } +} + +/// A mapping between a single remote reference and its advertised objects to a local destination which may or may not exist. +#[derive(Debug, Clone)] +pub struct Mapping { + /// The reference on the remote side, along with information about the objects they point to as advertised by the server. + pub remote: Source, + /// The local tracking reference to update after fetching the object visible via `remote`. + pub local: Option, + /// The index into the fetch ref-specs used to produce the mapping, allowing it to be recovered. + pub spec_index: usize, +} + +/// The status of the repository after the fetch operation +#[derive(Debug, Clone)] +pub enum Status { + /// Nothing changed as the remote didn't have anything new. + NoChange, + /// There was at least one tip with a new object which we received. + Change { + /// Information collected while writing the pack and its index. + write_pack_bundle: git_pack::bundle::write::Outcome, + /// Information collected while updating references. + update_refs: refs::update::Outcome, + }, +} + +/// The outcome of receiving a pack via [`Prepare::receive()`]. +#[derive(Debug, Clone)] +pub struct Outcome<'spec> { + /// The result of the initial mapping of references, the prerequisite for any fetch. + pub ref_map: RefMap<'spec>, + /// The status of the operation to indicate what happened. + pub status: Status, +} + +#[cfg(feature = "blocking-network-client")] +pub use super::connection::fetch::{negotiate, refs, Error, Prepare}; diff --git a/git-repository/src/remote/mod.rs b/git-repository/src/remote/mod.rs index d7db49e17dd..661ed9e5910 100644 --- a/git-repository/src/remote/mod.rs +++ b/git-repository/src/remote/mod.rs @@ -27,78 +27,7 @@ pub mod init; /// #[cfg(any(feature = "async-network-client", feature = "blocking-network-client"))] -pub mod fetch { - use crate::bstr::BString; - - /// Information about the relationship between our refspecs, and remote references with their local counterparts. - #[derive(Default, Debug, Clone)] - pub struct RefMap<'spec> { - /// A mapping between a remote reference and a local tracking branch. - pub mappings: Vec, - /// Information about the fixes applied to the `mapping` due to validation and sanitization. - pub fixes: Vec>, - /// All refs advertised by the remote. - pub remote_refs: Vec, - /// Additional information provided by the server as part of the handshake. - /// - /// Note that the `refs` field is always `None` as the refs are placed in `remote_refs`. - pub handshake: git_protocol::fetch::handshake::Outcome, - } - - /// Either an object id that the remote has or the matched remote ref itself. - #[derive(Debug, Clone)] - pub enum Source { - /// An object id, as the matched ref-spec was an object id itself. - ObjectId(git_hash::ObjectId), - /// The remote reference that matched the ref-specs name. - Ref(git_protocol::fetch::Ref), - } - - impl Source { - /// Return either the direct object id we refer to or the direct target that a reference refers to. - pub fn as_id(&self) -> &git_hash::oid { - match self { - Source::ObjectId(id) => id, - Source::Ref(r) => r.unpack().1, - } - } - } - - /// A mapping between a single remote reference and its advertised objects to a local destination which may or may not exist. - #[derive(Debug, Clone)] - pub struct Mapping { - /// The reference on the remote side, along with information about the objects they point to as advertised by the server. - pub remote: Source, - /// The local tracking reference to update after fetching the object visible via `remote`. - pub local: Option, - /// The index into the fetch ref-specs used to produce the mapping, allowing it to be recovered. - pub spec_index: usize, - } - - /// The status of the repository after the fetch operation - #[derive(Debug, Clone)] - pub enum Status { - /// Nothing changed as the remote didn't have anything new. - NoChange, - /// There was at least one tip with a new object which we received. - Change { - /// Information collected while writing the pack and its index. - write_pack_bundle: git_pack::bundle::write::Outcome, - }, - } - - /// The outcome of receiving a pack via [`Prepare::receive()`]. - #[derive(Debug, Clone)] - pub struct Outcome<'spec> { - /// The result of the initial mapping of references, the prerequisite for any fetch. - pub ref_map: RefMap<'spec>, - /// The status of the operation to indicate what happened. - pub status: Status, - } - - #[cfg(feature = "blocking-network-client")] - pub use super::connection::fetch::Prepare; -} +pub mod fetch; /// #[cfg(any(feature = "async-network-client", feature = "blocking-network-client"))] diff --git a/git-repository/tests/remote/fetch.rs b/git-repository/tests/remote/fetch.rs index 6f675699f76..701c9f3f01d 100644 --- a/git-repository/tests/remote/fetch.rs +++ b/git-repository/tests/remote/fetch.rs @@ -62,7 +62,10 @@ mod blocking_io { .prepare_fetch(Default::default())? .receive(&AtomicBool::default())?; match outcome.status { - fetch::Status::Change { write_pack_bundle } => { + fetch::Status::Change { + write_pack_bundle, + update_refs: _, // TODO: validate update refs + } => { assert_eq!(write_pack_bundle.pack_kind, git::odb::pack::data::Version::V2); assert_eq!(write_pack_bundle.object_hash, repo.object_hash()); assert_eq!(write_pack_bundle.index.num_objects, 3, "this value is 4 when git does it with 'consecutive' negotiation style, but could be 33 if completely naive."); From 4a5d3b4bb4f39a1e227da4ea77e96820cbed2e0d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 26 Sep 2022 17:43:30 +0800 Subject: [PATCH 070/113] make `remote::fetch::refs::update()` public to facilitate testing (#450) --- .../src/remote/connection/fetch/mod.rs | 4 ++-- .../fetch/{refs.rs => update_refs.rs} | 24 +++++++++++-------- 2 files changed, 16 insertions(+), 12 deletions(-) rename git-repository/src/remote/connection/fetch/{refs.rs => update_refs.rs} (78%) diff --git a/git-repository/src/remote/connection/fetch/mod.rs b/git-repository/src/remote/connection/fetch/mod.rs index 58a0b47d90c..0729c42582d 100644 --- a/git-repository/src/remote/connection/fetch/mod.rs +++ b/git-repository/src/remote/connection/fetch/mod.rs @@ -136,7 +136,6 @@ where iteration_mode: git_pack::data::input::Mode::Verify, object_hash: con.remote.repo.object_hash(), }; - let remote = con.remote; let write_pack_bundle = git_pack::Bundle::write_to_directory( reader, Some(repo.objects.store_ref().path().join("pack")), @@ -153,7 +152,7 @@ where git_protocol::fetch::indicate_end_of_interaction(&mut con.transport).ok(); } - let update_refs = refs::update(repo, remote.refspecs(crate::remote::Direction::Fetch), &self.ref_map)?; + let update_refs = refs::update(repo, &self.ref_map.mappings, false)?; Ok(Outcome { ref_map: std::mem::take(&mut self.ref_map), @@ -180,6 +179,7 @@ fn setup_remote_progress( mod config; /// +#[path = "update_refs.rs"] pub mod refs; /// A structure to hold the result of the handshake with the remote and configure the upcoming fetch operation. diff --git a/git-repository/src/remote/connection/fetch/refs.rs b/git-repository/src/remote/connection/fetch/update_refs.rs similarity index 78% rename from git-repository/src/remote/connection/fetch/refs.rs rename to git-repository/src/remote/connection/fetch/update_refs.rs index dddd7881c29..b34c06dcd04 100644 --- a/git-repository/src/remote/connection/fetch/refs.rs +++ b/git-repository/src/remote/connection/fetch/update_refs.rs @@ -1,4 +1,4 @@ -use crate::remote::fetch::RefMap; +use crate::remote::fetch; /// pub mod update { @@ -8,7 +8,7 @@ pub mod update { #[error("TBD")] pub struct Error {} } - use crate::remote::fetch::RefMap; + use crate::remote::fetch; pub use error::Error; /// The outcome of the refs-update operation at the end of a fetch. @@ -35,26 +35,25 @@ pub mod update { } impl Outcome { - /// Produce an iterator over all information used to produce the this outcome, ref-update by ref-update, using the `ref_map` + /// Produce an iterator over all information used to produce the this outcome, ref-update by ref-update, using the `mappings` /// used when producing the ref update. pub fn iter_mapping_updates<'a>( &self, - ref_map: &'a RefMap<'_>, + mappings: &'a [fetch::Mapping], ) -> impl Iterator< Item = ( &super::Update, - &'a crate::remote::fetch::Mapping, + &'a fetch::Mapping, Option<&git_ref::transaction::RefEdit>, ), > { self.updates .iter() - .zip(ref_map.mappings.iter()) + .zip(mappings.iter()) .map(move |(update, mapping)| (update, mapping, update.edit_index.and_then(|idx| self.edits.get(idx)))) } } } -use git_refspec::RefSpec; /// Information about the update of a single reference, corresponding the respective entry in [`RefMap::mapping`]. #[derive(Debug, Clone, Copy)] @@ -65,10 +64,15 @@ pub struct Update { pub edit_index: Option, } -pub(crate) fn update( +/// Update all refs as derived from `mappings` and produce an `Outcome` informing about all applied changes in detail. +/// If `dry_run` is true, ref transactions won't actually be applied, but are assumed to work without error so the underlying +/// `repo` is not actually changed. +/// +/// It can be used to produce typical information that one is used to from `git fetch`. +pub fn update( _repo: &crate::Repository, - _remote: &[RefSpec], - _ref_map: &RefMap<'_>, + _mappings: &[fetch::Mapping], + _dry_run: bool, ) -> Result { // TODO: tests and impl Ok(update::Outcome { From 8e1555d0ef0ea450979567c9aa9716c993e6320a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 26 Sep 2022 17:48:36 +0800 Subject: [PATCH 071/113] fix build (#450) --- .../src/remote/connection/fetch/mod.rs | 25 ++++++++++++++++++- .../remote/connection/fetch/update_refs.rs | 2 +- git-repository/src/remote/fetch.rs | 25 +------------------ 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/git-repository/src/remote/connection/fetch/mod.rs b/git-repository/src/remote/connection/fetch/mod.rs index 0729c42582d..12c69a85bfd 100644 --- a/git-repository/src/remote/connection/fetch/mod.rs +++ b/git-repository/src/remote/connection/fetch/mod.rs @@ -1,4 +1,4 @@ -use crate::remote::fetch::{Outcome, RefMap, Status}; +use crate::remote::fetch::RefMap; use crate::remote::{ref_map, Connection}; use crate::Progress; use git_odb::FindExt; @@ -30,6 +30,29 @@ mod error { } pub use error::Error; +/// The status of the repository after the fetch operation +#[derive(Debug, Clone)] +pub enum Status { + /// Nothing changed as the remote didn't have anything new. + NoChange, + /// There was at least one tip with a new object which we received. + Change { + /// Information collected while writing the pack and its index. + write_pack_bundle: git_pack::bundle::write::Outcome, + /// Information collected while updating references. + update_refs: refs::update::Outcome, + }, +} + +/// The outcome of receiving a pack via [`Prepare::receive()`]. +#[derive(Debug, Clone)] +pub struct Outcome<'spec> { + /// The result of the initial mapping of references, the prerequisite for any fetch. + pub ref_map: RefMap<'spec>, + /// The status of the operation to indicate what happened. + pub status: Status, +} + /// pub mod negotiate; diff --git a/git-repository/src/remote/connection/fetch/update_refs.rs b/git-repository/src/remote/connection/fetch/update_refs.rs index b34c06dcd04..89028a76f2c 100644 --- a/git-repository/src/remote/connection/fetch/update_refs.rs +++ b/git-repository/src/remote/connection/fetch/update_refs.rs @@ -55,7 +55,7 @@ pub mod update { } } -/// Information about the update of a single reference, corresponding the respective entry in [`RefMap::mapping`]. +/// Information about the update of a single reference, corresponding the respective entry in [`RefMap::mappings`][crate::remote::fetch::RefMap::mappings]. #[derive(Debug, Clone, Copy)] pub struct Update { /// The way the update was performed. diff --git a/git-repository/src/remote/fetch.rs b/git-repository/src/remote/fetch.rs index 682ce2195f6..1d635163e4e 100644 --- a/git-repository/src/remote/fetch.rs +++ b/git-repository/src/remote/fetch.rs @@ -45,28 +45,5 @@ pub struct Mapping { pub spec_index: usize, } -/// The status of the repository after the fetch operation -#[derive(Debug, Clone)] -pub enum Status { - /// Nothing changed as the remote didn't have anything new. - NoChange, - /// There was at least one tip with a new object which we received. - Change { - /// Information collected while writing the pack and its index. - write_pack_bundle: git_pack::bundle::write::Outcome, - /// Information collected while updating references. - update_refs: refs::update::Outcome, - }, -} - -/// The outcome of receiving a pack via [`Prepare::receive()`]. -#[derive(Debug, Clone)] -pub struct Outcome<'spec> { - /// The result of the initial mapping of references, the prerequisite for any fetch. - pub ref_map: RefMap<'spec>, - /// The status of the operation to indicate what happened. - pub status: Status, -} - #[cfg(feature = "blocking-network-client")] -pub use super::connection::fetch::{negotiate, refs, Error, Prepare}; +pub use super::connection::fetch::{negotiate, refs, Error, Outcome, Prepare, Status}; From c355823d405dbf8eb1287021895d5fb35a39e5f5 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 26 Sep 2022 20:17:12 +0800 Subject: [PATCH 072/113] the first somewhat synthetic test to check for no changes. (#450) This isn't super interesting as we run into this during integration testing, but testing it here would allow to test it only roughly later. --- .../remote/connection/fetch/update_refs.rs | 4 +- git-repository/tests/remote/fetch.rs | 92 ++++++++++++++++++- 2 files changed, 93 insertions(+), 3 deletions(-) diff --git a/git-repository/src/remote/connection/fetch/update_refs.rs b/git-repository/src/remote/connection/fetch/update_refs.rs index 89028a76f2c..fc438ef36de 100644 --- a/git-repository/src/remote/connection/fetch/update_refs.rs +++ b/git-repository/src/remote/connection/fetch/update_refs.rs @@ -23,7 +23,7 @@ pub mod update { } /// Describe the way a ref was updated - #[derive(Debug, Copy, Clone)] + #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub enum Mode { /// The old ref's commit was an ancestor of the new one, allowing for a fast-forward without a merge. FastForward, @@ -56,7 +56,7 @@ pub mod update { } /// Information about the update of a single reference, corresponding the respective entry in [`RefMap::mappings`][crate::remote::fetch::RefMap::mappings]. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Update { /// The way the update was performed. pub mode: update::Mode, diff --git a/git-repository/tests/remote/fetch.rs b/git-repository/tests/remote/fetch.rs index 701c9f3f01d..f2c90cf66f8 100644 --- a/git-repository/tests/remote/fetch.rs +++ b/git-repository/tests/remote/fetch.rs @@ -8,7 +8,7 @@ mod blocking_io { use git_testtools::hex_to_id; use std::sync::atomic::AtomicBool; - pub(crate) fn repo_rw(name: &str) -> (git::Repository, git_testtools::tempfile::TempDir) { + fn repo_rw(name: &str) -> (git::Repository, git_testtools::tempfile::TempDir) { let dir = git_testtools::scripted_fixture_repo_writable_with_args( "make_fetch_repos.sh", [git::path::realpath(remote::repo_path("base")) @@ -21,6 +21,96 @@ mod blocking_io { (repo, dir) } + mod refs { + use crate::remote; + use git_repository as git; + + fn repo(name: &str) -> git::Repository { + let dir = git_testtools::scripted_fixture_repo_read_only_with_args( + "make_fetch_repos.sh", + [git::path::realpath(remote::repo_path("base")) + .unwrap() + .to_string_lossy()], + ) + .unwrap(); + git::open_opts(dir.join(name), git::open::Options::isolated()).unwrap() + } + + mod update { + use crate::remote::fetch::blocking_io::refs::repo; + use git_ref::TargetRef; + use git_repository as git; + use git_repository::remote::fetch; + + #[test] + #[ignore] + fn remote_without_changes() { + let repo = repo("two-origins"); + let out = fetch::refs::update( + &repo, + &mapping_from_spec("refs/heads/main:refs/remotes/origin/main", &repo), + true, + ) + .unwrap(); + assert_eq!( + out.updates, + vec![fetch::refs::Update { + mode: fetch::refs::update::Mode::NoChangeNeeded, + edit_index: Some(0) + }] + ); + assert_eq!(out.edits.len(), 1); + } + + fn mapping_from_spec(spec: &str, repo: &git::Repository) -> Vec { + let spec = git_refspec::parse(spec.into(), git_refspec::parse::Operation::Fetch).unwrap(); + let group = git_refspec::MatchGroup::from_fetch_specs(Some(spec)); + let references = repo.references().unwrap(); + let references: Vec<_> = references.all().unwrap().map(|r| into_remote_ref(r.unwrap())).collect(); + group + .match_remotes(references.iter().map(remote_ref_to_item)) + .mappings + .into_iter() + .map(|m| fetch::Mapping { + remote: fetch::Source::Ref( + references[m.item_index.expect("set as all items are backed by ref")].clone(), + ), + local: m.rhs.map(|r| r.into_owned()), + spec_index: m.spec_index, + }) + .collect() + } + + fn into_remote_ref(mut r: git::Reference<'_>) -> git_protocol::fetch::Ref { + let full_ref_name = r.name().as_bstr().into(); + match r.target() { + TargetRef::Peeled(id) => git_protocol::fetch::Ref::Direct { + full_ref_name, + object: id.into(), + }, + TargetRef::Symbolic(name) => { + let target = name.as_bstr().into(); + let id = r.peel_to_id_in_place().unwrap(); + git_protocol::fetch::Ref::Symbolic { + full_ref_name, + target, + object: id.detach(), + } + } + } + } + + fn remote_ref_to_item(r: &git_protocol::fetch::Ref) -> git_refspec::match_group::Item<'_> { + let (full_ref_name, target, object) = r.unpack(); + git_refspec::match_group::Item { + full_ref_name, + target, + object, + } + } + } + } + #[test] fn fetch_pack() -> crate::Result { for version in [ From 2828674509f847528bb225c1a35e51efd7457c50 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 26 Sep 2022 20:27:07 +0800 Subject: [PATCH 073/113] more update tests (#450) --- .../remote/connection/fetch/update_refs.rs | 2 + git-repository/tests/remote/fetch.rs | 43 +++++++++++++++---- 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/git-repository/src/remote/connection/fetch/update_refs.rs b/git-repository/src/remote/connection/fetch/update_refs.rs index fc438ef36de..84b40948c35 100644 --- a/git-repository/src/remote/connection/fetch/update_refs.rs +++ b/git-repository/src/remote/connection/fetch/update_refs.rs @@ -29,6 +29,8 @@ pub mod update { FastForward, /// The ref was set to point to the new commit from the remote without taking into consideration its ancestry. Forced, + /// A new ref has been created as there was none before. + New, /// No change was attempted as the remote ref didn't change compared to the current ref, or because no remote ref was specified /// in the ref-spec. NoChangeNeeded, diff --git a/git-repository/tests/remote/fetch.rs b/git-repository/tests/remote/fetch.rs index f2c90cf66f8..215c11cb1b4 100644 --- a/git-repository/tests/remote/fetch.rs +++ b/git-repository/tests/remote/fetch.rs @@ -44,18 +44,45 @@ mod blocking_io { #[test] #[ignore] - fn remote_without_changes() { + fn various_valid_updates() { let repo = repo("two-origins"); - let out = fetch::refs::update( - &repo, - &mapping_from_spec("refs/heads/main:refs/remotes/origin/main", &repo), - true, - ) - .unwrap(); + for (spec, expected_mode) in [ + ( + "refs/heads/main:refs/remotes/origin/main", + fetch::refs::update::Mode::NoChangeNeeded, + ), + ( + "refs/heads/main:refs/remotes/origin/new-main", + fetch::refs::update::Mode::New, + ), + ("+refs/heads/main:refs/heads/g", fetch::refs::update::Mode::Forced), + ] { + let out = fetch::refs::update(&repo, &mapping_from_spec(spec, &repo), true).unwrap(); + + assert_eq!( + out.updates, + vec![fetch::refs::Update { + mode: expected_mode, + edit_index: Some(0) + }] + ); + assert_eq!(out.edits.len(), 1); + } + } + + #[test] + #[ignore] + #[should_panic] + fn fast_forward_is_not_implemented_yet() { + // TODO: move it above for acceptable case, test here for non-fastforwards being denied. + let repo = repo("two-origins"); + let out = fetch::refs::update(&repo, &mapping_from_spec("+refs/heads/main:refs/heads/g", &repo), true) + .unwrap(); + assert_eq!( out.updates, vec![fetch::refs::Update { - mode: fetch::refs::update::Mode::NoChangeNeeded, + mode: fetch::refs::update::Mode::FastForward, edit_index: Some(0) }] ); From 658c1257c073507327d9a50c1c89b49d17e9ccbc Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 26 Sep 2022 22:09:36 +0800 Subject: [PATCH 074/113] feat: `FullName::try_from(&BString)` for convenience. (#450) Sometimes when matching one only has a `&BString`, and it's hard to convert it to `&BStr` without an extra line of code, it's cumbersome, so we workaround by adding another conversion. --- git-ref/src/fullname.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/git-ref/src/fullname.rs b/git-ref/src/fullname.rs index 053b67ee77a..e8ed206a16f 100644 --- a/git-ref/src/fullname.rs +++ b/git-ref/src/fullname.rs @@ -38,6 +38,15 @@ impl TryFrom for FullName { } } +impl TryFrom<&BString> for FullName { + type Error = git_validate::refname::Error; + + fn try_from(value: &BString) -> Result { + git_validate::refname(value.as_ref())?; + Ok(FullName(value.clone())) + } +} + impl From for BString { fn from(name: FullName) -> Self { name.0 From c101d50c315d922885309e5939a10853c553eb68 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 26 Sep 2022 22:11:02 +0800 Subject: [PATCH 075/113] a big step towards ref updates, now it needs specs (#450) --- .../src/remote/connection/fetch/mod.rs | 4 +- .../remote/connection/fetch/update_refs.rs | 93 ++++++++++++++++--- git-repository/src/remote/fetch.rs | 9 ++ git-repository/tests/remote/fetch.rs | 32 +++++-- 4 files changed, 116 insertions(+), 22 deletions(-) diff --git a/git-repository/src/remote/connection/fetch/mod.rs b/git-repository/src/remote/connection/fetch/mod.rs index 12c69a85bfd..4fd9375b22a 100644 --- a/git-repository/src/remote/connection/fetch/mod.rs +++ b/git-repository/src/remote/connection/fetch/mod.rs @@ -1,5 +1,5 @@ use crate::remote::fetch::RefMap; -use crate::remote::{ref_map, Connection}; +use crate::remote::{fetch, ref_map, Connection}; use crate::Progress; use git_odb::FindExt; use git_protocol::transport::client::Transport; @@ -175,7 +175,7 @@ where git_protocol::fetch::indicate_end_of_interaction(&mut con.transport).ok(); } - let update_refs = refs::update(repo, &self.ref_map.mappings, false)?; + let update_refs = refs::update(repo, &self.ref_map.mappings, fetch::DryRun::No)?; Ok(Outcome { ref_map: std::mem::take(&mut self.ref_map), diff --git a/git-repository/src/remote/connection/fetch/update_refs.rs b/git-repository/src/remote/connection/fetch/update_refs.rs index 84b40948c35..ed93006a5e6 100644 --- a/git-repository/src/remote/connection/fetch/update_refs.rs +++ b/git-repository/src/remote/connection/fetch/update_refs.rs @@ -1,14 +1,22 @@ use crate::remote::fetch; +use git_ref::transaction::{Change, LogChange, PreviousValue, RefEdit, RefLog}; +use git_ref::{Target, TargetRef}; +use std::convert::TryInto; /// pub mod update { + use crate::remote::fetch; mod error { - /// The error returned when updating refs after a fetch operation. + /// The error returned by [`fetch::refs::update()`]. #[derive(Debug, thiserror::Error)] - #[error("TBD")] - pub struct Error {} + #[allow(missing_docs)] + pub enum Error { + #[error(transparent)] + FindReference(#[from] crate::reference::find::Error), + #[error("A remote reference had a name that wasn't considered valid. Corrupt remote repo or insufficient checks on remote?")] + InvalidRefName(#[from] git_validate::refname::Error), + } } - use crate::remote::fetch; pub use error::Error; /// The outcome of the refs-update operation at the end of a fetch. @@ -31,6 +39,10 @@ pub mod update { Forced, /// A new ref has been created as there was none before. New, + /// The reference update would not have been a fast-forward, and force is not specified in the ref-spec. + RejectedNonFastForward, + /// The update of a local symbolic reference was rejected. + RejectedSymbolic, /// No change was attempted as the remote ref didn't change compared to the current ref, or because no remote ref was specified /// in the ref-spec. NoChangeNeeded, @@ -64,6 +76,8 @@ pub struct Update { pub mode: update::Mode, /// The index to the edit that was created from the corresponding mapping, or `None` if there was no local ref. pub edit_index: Option, + /// The index of the ref-spec from which the source mapping originated. + pub spec_index: usize, } /// Update all refs as derived from `mappings` and produce an `Outcome` informing about all applied changes in detail. @@ -72,13 +86,68 @@ pub struct Update { /// /// It can be used to produce typical information that one is used to from `git fetch`. pub fn update( - _repo: &crate::Repository, - _mappings: &[fetch::Mapping], - _dry_run: bool, + repo: &crate::Repository, + mappings: &[fetch::Mapping], + _dry_run: fetch::DryRun, ) -> Result { - // TODO: tests and impl - Ok(update::Outcome { - edits: Default::default(), - updates: Default::default(), - }) + let mut edits = Vec::new(); + let mut updates = Vec::new(); + + for fetch::Mapping { + remote, + local, + spec_index, + } in mappings + { + let remote_id = remote.as_id(); + let (mode, edit_index) = match local { + Some(name) => { + let (mode, reflog_message, name) = match repo.try_find_reference(name)? { + Some(existing) => match existing.target() { + TargetRef::Symbolic(_) => { + updates.push(Update { + mode: update::Mode::RejectedSymbolic, + spec_index: *spec_index, + edit_index: None, + }); + continue; + } + TargetRef::Peeled(local_id) => { + let (mode, reflog_message) = if local_id == remote_id { + (update::Mode::NoChangeNeeded, "TBD no change") + } else { + todo!("determine fast forward or force") + }; + (mode, reflog_message, existing.name().to_owned()) + } + }, + None => (update::Mode::New, "TBD new", name.try_into()?), + }; + let edit = RefEdit { + change: Change::Update { + log: LogChange { + mode: RefLog::AndReference, + force_create_reflog: false, + message: reflog_message.into(), + }, + expected: PreviousValue::ExistingMustMatch(Target::Peeled(remote_id.into())), + new: Target::Peeled(remote_id.into()), + }, + name, + deref: false, + }; + let edit_index = edits.len(); + edits.push(edit); + (mode, Some(edit_index)) + } + None => (update::Mode::NoChangeNeeded, None), + }; + updates.push(Update { + mode, + spec_index: *spec_index, + edit_index, + }) + } + + Ok(update::Outcome { edits, updates }) } diff --git a/git-repository/src/remote/fetch.rs b/git-repository/src/remote/fetch.rs index 1d635163e4e..db154ff2abb 100644 --- a/git-repository/src/remote/fetch.rs +++ b/git-repository/src/remote/fetch.rs @@ -1,5 +1,14 @@ use crate::bstr::BString; +/// If `Yes`, don't really make changes but do as much as possible to get an idea of what would be done. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum DryRun { + /// Enable dry-run mode and don't actually change the underlying repository in any way. + Yes, + /// Run the operation like normal, making changes to the underlying repository. + No, +} + /// Information about the relationship between our refspecs, and remote references with their local counterparts. #[derive(Default, Debug, Clone)] pub struct RefMap<'spec> { diff --git a/git-repository/tests/remote/fetch.rs b/git-repository/tests/remote/fetch.rs index 215c11cb1b4..932a7cbf832 100644 --- a/git-repository/tests/remote/fetch.rs +++ b/git-repository/tests/remote/fetch.rs @@ -46,25 +46,36 @@ mod blocking_io { #[ignore] fn various_valid_updates() { let repo = repo("two-origins"); - for (spec, expected_mode) in [ + // TODO: test reflog message (various cases if it's new) + for (spec, expected_mode, detail) in [ ( "refs/heads/main:refs/remotes/origin/main", fetch::refs::update::Mode::NoChangeNeeded, + "these refs are en-par since the initial clone" + ), + ( + "refs/heads/main", + fetch::refs::update::Mode::NoChangeNeeded, + "without local destination ref there is nothing to do for us, ever (except for FETCH_HEADs) later" ), ( "refs/heads/main:refs/remotes/origin/new-main", fetch::refs::update::Mode::New, + "the destination branch doesn't exist and needs to be created" ), - ("+refs/heads/main:refs/heads/g", fetch::refs::update::Mode::Forced), + ("+refs/heads/main:refs/heads/g", fetch::refs::update::Mode::Forced, "a forced non-fastforward (main goes backwards)"), + // ("refs/heads/g:refs/heads/main", fetch::refs::update::Mode::FastForward, "a fast-forward only fast-forward situation, all good"), ] { - let out = fetch::refs::update(&repo, &mapping_from_spec(spec, &repo), true).unwrap(); + let out = fetch::refs::update(&repo, &mapping_from_spec(spec, &repo), fetch::DryRun::Yes).unwrap(); assert_eq!( out.updates, vec![fetch::refs::Update { mode: expected_mode, - edit_index: Some(0) - }] + edit_index: Some(0), + spec_index: 0 + }], + "{spec:?}: {detail}" ); assert_eq!(out.edits.len(), 1); } @@ -76,14 +87,19 @@ mod blocking_io { fn fast_forward_is_not_implemented_yet() { // TODO: move it above for acceptable case, test here for non-fastforwards being denied. let repo = repo("two-origins"); - let out = fetch::refs::update(&repo, &mapping_from_spec("+refs/heads/main:refs/heads/g", &repo), true) - .unwrap(); + let out = fetch::refs::update( + &repo, + &mapping_from_spec("+refs/heads/main:refs/heads/g", &repo), + fetch::DryRun::Yes, + ) + .unwrap(); assert_eq!( out.updates, vec![fetch::refs::Update { mode: fetch::refs::update::Mode::FastForward, - edit_index: Some(0) + edit_index: Some(0), + spec_index: 0, }] ); assert_eq!(out.edits.len(), 1); From d7f63a6c60a826dc862bd13adbef041e4ac6d8ab Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 26 Sep 2022 22:20:38 +0800 Subject: [PATCH 076/113] feat: `RefSpec::allow_non_fast_forward()` to get information about 'force' quickly. (#450) --- git-refspec/src/spec.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/git-refspec/src/spec.rs b/git-refspec/src/spec.rs index ba5b7932698..efd97abd3fe 100644 --- a/git-refspec/src/spec.rs +++ b/git-refspec/src/spec.rs @@ -18,6 +18,11 @@ impl RefSpec { dst: self.dst.as_ref().map(|b| b.as_ref()), } } + + /// Return true if the spec stats with a `+` and thus forces setting the reference. + pub fn allow_non_fast_forward(&self) -> bool { + matches!(self.mode, Mode::Force) + } } mod impls { From a9f2c458c1858e8d40ed0efdc762f78b9efb7783 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Mon, 26 Sep 2022 22:21:19 +0800 Subject: [PATCH 077/113] Provide refspecs to refs::update() to obtain force information (#450) --- .../src/remote/connection/fetch/mod.rs | 7 ++++++- .../remote/connection/fetch/update_refs.rs | 7 +++++-- git-repository/tests/remote/fetch.rs | 21 ++++++++++--------- 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/git-repository/src/remote/connection/fetch/mod.rs b/git-repository/src/remote/connection/fetch/mod.rs index 4fd9375b22a..3b0c5be153f 100644 --- a/git-repository/src/remote/connection/fetch/mod.rs +++ b/git-repository/src/remote/connection/fetch/mod.rs @@ -175,7 +175,12 @@ where git_protocol::fetch::indicate_end_of_interaction(&mut con.transport).ok(); } - let update_refs = refs::update(repo, &self.ref_map.mappings, fetch::DryRun::No)?; + let update_refs = refs::update( + repo, + &self.ref_map.mappings, + con.remote.refspecs(crate::remote::Direction::Fetch), + fetch::DryRun::No, + )?; Ok(Outcome { ref_map: std::mem::take(&mut self.ref_map), diff --git a/git-repository/src/remote/connection/fetch/update_refs.rs b/git-repository/src/remote/connection/fetch/update_refs.rs index ed93006a5e6..6ecd38843e0 100644 --- a/git-repository/src/remote/connection/fetch/update_refs.rs +++ b/git-repository/src/remote/connection/fetch/update_refs.rs @@ -7,7 +7,7 @@ use std::convert::TryInto; pub mod update { use crate::remote::fetch; mod error { - /// The error returned by [`fetch::refs::update()`]. + /// The error returned by [`fetch::refs::update()`][`crate::remote::fetch::refs::update()`]. #[derive(Debug, thiserror::Error)] #[allow(missing_docs)] pub enum Error { @@ -88,6 +88,7 @@ pub struct Update { pub fn update( repo: &crate::Repository, mappings: &[fetch::Mapping], + refspecs: &[git_refspec::RefSpec], _dry_run: fetch::DryRun, ) -> Result { let mut edits = Vec::new(); @@ -115,8 +116,10 @@ pub fn update( TargetRef::Peeled(local_id) => { let (mode, reflog_message) = if local_id == remote_id { (update::Mode::NoChangeNeeded, "TBD no change") + } else if refspecs[*spec_index].allow_non_fast_forward() { + (update::Mode::Forced, "TBD force") } else { - todo!("determine fast forward or force") + todo!("check for fast-forward (is local an ancestor of remote?)") }; (mode, reflog_message, existing.name().to_owned()) } diff --git a/git-repository/tests/remote/fetch.rs b/git-repository/tests/remote/fetch.rs index 932a7cbf832..e7dae25ccda 100644 --- a/git-repository/tests/remote/fetch.rs +++ b/git-repository/tests/remote/fetch.rs @@ -66,7 +66,8 @@ mod blocking_io { ("+refs/heads/main:refs/heads/g", fetch::refs::update::Mode::Forced, "a forced non-fastforward (main goes backwards)"), // ("refs/heads/g:refs/heads/main", fetch::refs::update::Mode::FastForward, "a fast-forward only fast-forward situation, all good"), ] { - let out = fetch::refs::update(&repo, &mapping_from_spec(spec, &repo), fetch::DryRun::Yes).unwrap(); + let (mapping, specs) = mapping_from_spec(spec, &repo); + let out = fetch::refs::update(&repo, &mapping, &specs, fetch::DryRun::Yes).unwrap(); assert_eq!( out.updates, @@ -87,12 +88,8 @@ mod blocking_io { fn fast_forward_is_not_implemented_yet() { // TODO: move it above for acceptable case, test here for non-fastforwards being denied. let repo = repo("two-origins"); - let out = fetch::refs::update( - &repo, - &mapping_from_spec("+refs/heads/main:refs/heads/g", &repo), - fetch::DryRun::Yes, - ) - .unwrap(); + let (mappings, specs) = mapping_from_spec("+refs/heads/main:refs/heads/g", &repo); + let out = fetch::refs::update(&repo, &mappings, &specs, fetch::DryRun::Yes).unwrap(); assert_eq!( out.updates, @@ -105,12 +102,15 @@ mod blocking_io { assert_eq!(out.edits.len(), 1); } - fn mapping_from_spec(spec: &str, repo: &git::Repository) -> Vec { + fn mapping_from_spec( + spec: &str, + repo: &git::Repository, + ) -> (Vec, Vec) { let spec = git_refspec::parse(spec.into(), git_refspec::parse::Operation::Fetch).unwrap(); let group = git_refspec::MatchGroup::from_fetch_specs(Some(spec)); let references = repo.references().unwrap(); let references: Vec<_> = references.all().unwrap().map(|r| into_remote_ref(r.unwrap())).collect(); - group + let mappings = group .match_remotes(references.iter().map(remote_ref_to_item)) .mappings .into_iter() @@ -121,7 +121,8 @@ mod blocking_io { local: m.rhs.map(|r| r.into_owned()), spec_index: m.spec_index, }) - .collect() + .collect(); + (mappings, vec![spec.to_owned()]) } fn into_remote_ref(mut r: git::Reference<'_>) -> git_protocol::fetch::Ref { From 9f9b61070d0b6e10e795e1401401d55c554c59b9 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 27 Sep 2022 07:32:11 +0800 Subject: [PATCH 078/113] Make `fetch::refs::update()` private again, move tests accordingly. (#450) It's so hard to call I doubt it should ever be called manually outside of the confines of its test-suite. --- .../src/remote/connection/fetch/mod.rs | 2 +- .../{update_refs.rs => update_refs/mod.rs} | 70 +-------- .../connection/fetch/update_refs/tests.rs | 134 ++++++++++++++++++ .../connection/fetch/update_refs/update.rs | 64 +++++++++ git-repository/src/remote/fetch.rs | 3 +- git-repository/tests/remote/fetch.rs | 134 ------------------ 6 files changed, 206 insertions(+), 201 deletions(-) rename git-repository/src/remote/connection/fetch/{update_refs.rs => update_refs/mod.rs} (57%) create mode 100644 git-repository/src/remote/connection/fetch/update_refs/tests.rs create mode 100644 git-repository/src/remote/connection/fetch/update_refs/update.rs diff --git a/git-repository/src/remote/connection/fetch/mod.rs b/git-repository/src/remote/connection/fetch/mod.rs index 3b0c5be153f..41381169ee9 100644 --- a/git-repository/src/remote/connection/fetch/mod.rs +++ b/git-repository/src/remote/connection/fetch/mod.rs @@ -207,7 +207,7 @@ fn setup_remote_progress( mod config; /// -#[path = "update_refs.rs"] +#[path = "update_refs/mod.rs"] pub mod refs; /// A structure to hold the result of the handshake with the remote and configure the upcoming fetch operation. diff --git a/git-repository/src/remote/connection/fetch/update_refs.rs b/git-repository/src/remote/connection/fetch/update_refs/mod.rs similarity index 57% rename from git-repository/src/remote/connection/fetch/update_refs.rs rename to git-repository/src/remote/connection/fetch/update_refs/mod.rs index 6ecd38843e0..73fcdf96484 100644 --- a/git-repository/src/remote/connection/fetch/update_refs.rs +++ b/git-repository/src/remote/connection/fetch/update_refs/mod.rs @@ -4,70 +4,7 @@ use git_ref::{Target, TargetRef}; use std::convert::TryInto; /// -pub mod update { - use crate::remote::fetch; - mod error { - /// The error returned by [`fetch::refs::update()`][`crate::remote::fetch::refs::update()`]. - #[derive(Debug, thiserror::Error)] - #[allow(missing_docs)] - pub enum Error { - #[error(transparent)] - FindReference(#[from] crate::reference::find::Error), - #[error("A remote reference had a name that wasn't considered valid. Corrupt remote repo or insufficient checks on remote?")] - InvalidRefName(#[from] git_validate::refname::Error), - } - } - pub use error::Error; - - /// The outcome of the refs-update operation at the end of a fetch. - #[derive(Debug, Clone)] - pub struct Outcome { - /// All edits that were performed to update local refs. - pub edits: Vec, - /// Each update provides more information about what happened to the corresponding mapping. - /// Use [`iter_mapping_updates()`][Self::iter_mapping_updates()] to recombine the update information with ref-edits and their - /// mapping. - pub updates: Vec, - } - - /// Describe the way a ref was updated - #[derive(Debug, Copy, Clone, PartialEq, Eq)] - pub enum Mode { - /// The old ref's commit was an ancestor of the new one, allowing for a fast-forward without a merge. - FastForward, - /// The ref was set to point to the new commit from the remote without taking into consideration its ancestry. - Forced, - /// A new ref has been created as there was none before. - New, - /// The reference update would not have been a fast-forward, and force is not specified in the ref-spec. - RejectedNonFastForward, - /// The update of a local symbolic reference was rejected. - RejectedSymbolic, - /// No change was attempted as the remote ref didn't change compared to the current ref, or because no remote ref was specified - /// in the ref-spec. - NoChangeNeeded, - } - - impl Outcome { - /// Produce an iterator over all information used to produce the this outcome, ref-update by ref-update, using the `mappings` - /// used when producing the ref update. - pub fn iter_mapping_updates<'a>( - &self, - mappings: &'a [fetch::Mapping], - ) -> impl Iterator< - Item = ( - &super::Update, - &'a fetch::Mapping, - Option<&git_ref::transaction::RefEdit>, - ), - > { - self.updates - .iter() - .zip(mappings.iter()) - .map(move |(update, mapping)| (update, mapping, update.edit_index.and_then(|idx| self.edits.get(idx)))) - } - } -} +pub mod update; /// Information about the update of a single reference, corresponding the respective entry in [`RefMap::mappings`][crate::remote::fetch::RefMap::mappings]. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -85,7 +22,7 @@ pub struct Update { /// `repo` is not actually changed. /// /// It can be used to produce typical information that one is used to from `git fetch`. -pub fn update( +pub(crate) fn update( repo: &crate::Repository, mappings: &[fetch::Mapping], refspecs: &[git_refspec::RefSpec], @@ -154,3 +91,6 @@ pub fn update( Ok(update::Outcome { edits, updates }) } + +#[cfg(test)] +mod tests; diff --git a/git-repository/src/remote/connection/fetch/update_refs/tests.rs b/git-repository/src/remote/connection/fetch/update_refs/tests.rs new file mode 100644 index 00000000000..e25979e2e6a --- /dev/null +++ b/git-repository/src/remote/connection/fetch/update_refs/tests.rs @@ -0,0 +1,134 @@ +mod update { + + use crate as git; + + fn repo(name: &str) -> git::Repository { + let dir = git_testtools::scripted_fixture_repo_read_only_with_args( + "make_fetch_repos.sh", + [git::path::realpath( + git_testtools::scripted_fixture_repo_read_only("make_remote_repos.sh") + .unwrap() + .join("base"), + ) + .unwrap() + .to_string_lossy()], + ) + .unwrap(); + git::open_opts(dir.join(name), git::open::Options::isolated()).unwrap() + } + + use crate::remote::fetch; + use git_ref::TargetRef; + + #[test] + #[ignore] + fn various_valid_updates() { + let repo = repo("two-origins"); + // TODO: test reflog message (various cases if it's new) + for (spec, expected_mode, detail) in [ + ( + "refs/heads/main:refs/remotes/origin/main", + fetch::refs::update::Mode::NoChangeNeeded, + "these refs are en-par since the initial clone", + ), + ( + "refs/heads/main", + fetch::refs::update::Mode::NoChangeNeeded, + "without local destination ref there is nothing to do for us, ever (except for FETCH_HEADs) later", + ), + ( + "refs/heads/main:refs/remotes/origin/new-main", + fetch::refs::update::Mode::New, + "the destination branch doesn't exist and needs to be created", + ), + ( + "+refs/heads/main:refs/heads/g", + fetch::refs::update::Mode::Forced, + "a forced non-fastforward (main goes backwards)", + ), + // ("refs/heads/g:refs/heads/main", fetch::refs::update::Mode::FastForward, "a fast-forward only fast-forward situation, all good"), + ] { + let (mapping, specs) = mapping_from_spec(spec, &repo); + let out = fetch::refs::update(&repo, &mapping, &specs, fetch::DryRun::Yes).unwrap(); + + assert_eq!( + out.updates, + vec![fetch::refs::Update { + mode: expected_mode, + edit_index: Some(0), + spec_index: 0 + }], + "{spec:?}: {detail}" + ); + assert_eq!(out.edits.len(), 1); + } + } + + #[test] + #[ignore] + #[should_panic] + fn fast_forward_is_not_implemented_yet() { + // TODO: move it above for acceptable case, test here for non-fastforwards being denied. + let repo = repo("two-origins"); + let (mappings, specs) = mapping_from_spec("+refs/heads/main:refs/heads/g", &repo); + let out = fetch::refs::update(&repo, &mappings, &specs, fetch::DryRun::Yes).unwrap(); + + assert_eq!( + out.updates, + vec![fetch::refs::Update { + mode: fetch::refs::update::Mode::FastForward, + edit_index: Some(0), + spec_index: 0, + }] + ); + assert_eq!(out.edits.len(), 1); + } + + fn mapping_from_spec(spec: &str, repo: &git::Repository) -> (Vec, Vec) { + let spec = git_refspec::parse(spec.into(), git_refspec::parse::Operation::Fetch).unwrap(); + let group = git_refspec::MatchGroup::from_fetch_specs(Some(spec)); + let references = repo.references().unwrap(); + let references: Vec<_> = references.all().unwrap().map(|r| into_remote_ref(r.unwrap())).collect(); + let mappings = group + .match_remotes(references.iter().map(remote_ref_to_item)) + .mappings + .into_iter() + .map(|m| fetch::Mapping { + remote: fetch::Source::Ref( + references[m.item_index.expect("set as all items are backed by ref")].clone(), + ), + local: m.rhs.map(|r| r.into_owned()), + spec_index: m.spec_index, + }) + .collect(); + (mappings, vec![spec.to_owned()]) + } + + fn into_remote_ref(mut r: git::Reference<'_>) -> git_protocol::fetch::Ref { + let full_ref_name = r.name().as_bstr().into(); + match r.target() { + TargetRef::Peeled(id) => git_protocol::fetch::Ref::Direct { + full_ref_name, + object: id.into(), + }, + TargetRef::Symbolic(name) => { + let target = name.as_bstr().into(); + let id = r.peel_to_id_in_place().unwrap(); + git_protocol::fetch::Ref::Symbolic { + full_ref_name, + target, + object: id.detach(), + } + } + } + } + + fn remote_ref_to_item(r: &git_protocol::fetch::Ref) -> git_refspec::match_group::Item<'_> { + let (full_ref_name, target, object) = r.unpack(); + git_refspec::match_group::Item { + full_ref_name, + target, + object, + } + } +} diff --git a/git-repository/src/remote/connection/fetch/update_refs/update.rs b/git-repository/src/remote/connection/fetch/update_refs/update.rs new file mode 100644 index 00000000000..eebe02f49a7 --- /dev/null +++ b/git-repository/src/remote/connection/fetch/update_refs/update.rs @@ -0,0 +1,64 @@ +use crate::remote::fetch; + +mod error { + /// The error returned when updating references. + #[derive(Debug, thiserror::Error)] + #[allow(missing_docs)] + pub enum Error { + #[error(transparent)] + FindReference(#[from] crate::reference::find::Error), + #[error("A remote reference had a name that wasn't considered valid. Corrupt remote repo or insufficient checks on remote?")] + InvalidRefName(#[from] git_validate::refname::Error), + } +} + +pub use error::Error; + +/// The outcome of the refs-update operation at the end of a fetch. +#[derive(Debug, Clone)] +pub struct Outcome { + /// All edits that were performed to update local refs. + pub edits: Vec, + /// Each update provides more information about what happened to the corresponding mapping. + /// Use [`iter_mapping_updates()`][Self::iter_mapping_updates()] to recombine the update information with ref-edits and their + /// mapping. + pub updates: Vec, +} + +/// Describe the way a ref was updated +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum Mode { + /// The old ref's commit was an ancestor of the new one, allowing for a fast-forward without a merge. + FastForward, + /// The ref was set to point to the new commit from the remote without taking into consideration its ancestry. + Forced, + /// A new ref has been created as there was none before. + New, + /// The reference update would not have been a fast-forward, and force is not specified in the ref-spec. + RejectedNonFastForward, + /// The update of a local symbolic reference was rejected. + RejectedSymbolic, + /// No change was attempted as the remote ref didn't change compared to the current ref, or because no remote ref was specified + /// in the ref-spec. + NoChangeNeeded, +} + +impl Outcome { + /// Produce an iterator over all information used to produce the this outcome, ref-update by ref-update, using the `mappings` + /// used when producing the ref update. + pub fn iter_mapping_updates<'a>( + &self, + mappings: &'a [fetch::Mapping], + ) -> impl Iterator< + Item = ( + &super::Update, + &'a fetch::Mapping, + Option<&git_ref::transaction::RefEdit>, + ), + > { + self.updates + .iter() + .zip(mappings.iter()) + .map(move |(update, mapping)| (update, mapping, update.edit_index.and_then(|idx| self.edits.get(idx)))) + } +} diff --git a/git-repository/src/remote/fetch.rs b/git-repository/src/remote/fetch.rs index db154ff2abb..7edb14f7683 100644 --- a/git-repository/src/remote/fetch.rs +++ b/git-repository/src/remote/fetch.rs @@ -2,8 +2,9 @@ use crate::bstr::BString; /// If `Yes`, don't really make changes but do as much as possible to get an idea of what would be done. #[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub enum DryRun { +pub(crate) enum DryRun { /// Enable dry-run mode and don't actually change the underlying repository in any way. + #[cfg(test)] Yes, /// Run the operation like normal, making changes to the underlying repository. No, diff --git a/git-repository/tests/remote/fetch.rs b/git-repository/tests/remote/fetch.rs index e7dae25ccda..6a04c9dc511 100644 --- a/git-repository/tests/remote/fetch.rs +++ b/git-repository/tests/remote/fetch.rs @@ -21,140 +21,6 @@ mod blocking_io { (repo, dir) } - mod refs { - use crate::remote; - use git_repository as git; - - fn repo(name: &str) -> git::Repository { - let dir = git_testtools::scripted_fixture_repo_read_only_with_args( - "make_fetch_repos.sh", - [git::path::realpath(remote::repo_path("base")) - .unwrap() - .to_string_lossy()], - ) - .unwrap(); - git::open_opts(dir.join(name), git::open::Options::isolated()).unwrap() - } - - mod update { - use crate::remote::fetch::blocking_io::refs::repo; - use git_ref::TargetRef; - use git_repository as git; - use git_repository::remote::fetch; - - #[test] - #[ignore] - fn various_valid_updates() { - let repo = repo("two-origins"); - // TODO: test reflog message (various cases if it's new) - for (spec, expected_mode, detail) in [ - ( - "refs/heads/main:refs/remotes/origin/main", - fetch::refs::update::Mode::NoChangeNeeded, - "these refs are en-par since the initial clone" - ), - ( - "refs/heads/main", - fetch::refs::update::Mode::NoChangeNeeded, - "without local destination ref there is nothing to do for us, ever (except for FETCH_HEADs) later" - ), - ( - "refs/heads/main:refs/remotes/origin/new-main", - fetch::refs::update::Mode::New, - "the destination branch doesn't exist and needs to be created" - ), - ("+refs/heads/main:refs/heads/g", fetch::refs::update::Mode::Forced, "a forced non-fastforward (main goes backwards)"), - // ("refs/heads/g:refs/heads/main", fetch::refs::update::Mode::FastForward, "a fast-forward only fast-forward situation, all good"), - ] { - let (mapping, specs) = mapping_from_spec(spec, &repo); - let out = fetch::refs::update(&repo, &mapping, &specs, fetch::DryRun::Yes).unwrap(); - - assert_eq!( - out.updates, - vec![fetch::refs::Update { - mode: expected_mode, - edit_index: Some(0), - spec_index: 0 - }], - "{spec:?}: {detail}" - ); - assert_eq!(out.edits.len(), 1); - } - } - - #[test] - #[ignore] - #[should_panic] - fn fast_forward_is_not_implemented_yet() { - // TODO: move it above for acceptable case, test here for non-fastforwards being denied. - let repo = repo("two-origins"); - let (mappings, specs) = mapping_from_spec("+refs/heads/main:refs/heads/g", &repo); - let out = fetch::refs::update(&repo, &mappings, &specs, fetch::DryRun::Yes).unwrap(); - - assert_eq!( - out.updates, - vec![fetch::refs::Update { - mode: fetch::refs::update::Mode::FastForward, - edit_index: Some(0), - spec_index: 0, - }] - ); - assert_eq!(out.edits.len(), 1); - } - - fn mapping_from_spec( - spec: &str, - repo: &git::Repository, - ) -> (Vec, Vec) { - let spec = git_refspec::parse(spec.into(), git_refspec::parse::Operation::Fetch).unwrap(); - let group = git_refspec::MatchGroup::from_fetch_specs(Some(spec)); - let references = repo.references().unwrap(); - let references: Vec<_> = references.all().unwrap().map(|r| into_remote_ref(r.unwrap())).collect(); - let mappings = group - .match_remotes(references.iter().map(remote_ref_to_item)) - .mappings - .into_iter() - .map(|m| fetch::Mapping { - remote: fetch::Source::Ref( - references[m.item_index.expect("set as all items are backed by ref")].clone(), - ), - local: m.rhs.map(|r| r.into_owned()), - spec_index: m.spec_index, - }) - .collect(); - (mappings, vec![spec.to_owned()]) - } - - fn into_remote_ref(mut r: git::Reference<'_>) -> git_protocol::fetch::Ref { - let full_ref_name = r.name().as_bstr().into(); - match r.target() { - TargetRef::Peeled(id) => git_protocol::fetch::Ref::Direct { - full_ref_name, - object: id.into(), - }, - TargetRef::Symbolic(name) => { - let target = name.as_bstr().into(); - let id = r.peel_to_id_in_place().unwrap(); - git_protocol::fetch::Ref::Symbolic { - full_ref_name, - target, - object: id.detach(), - } - } - } - } - - fn remote_ref_to_item(r: &git_protocol::fetch::Ref) -> git_refspec::match_group::Item<'_> { - let (full_ref_name, target, object) = r.unpack(); - git_refspec::match_group::Item { - full_ref_name, - target, - object, - } - } - } - } - #[test] fn fetch_pack() -> crate::Result { for version in [ From e4edc1897cc8ffa0dfd0c34cfaf6eb2f9d5b86c6 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 27 Sep 2022 07:46:11 +0800 Subject: [PATCH 079/113] the first successful test (#450) --- .../connection/fetch/update_refs/tests.rs | 20 +++++++++++++------ .../tests/fixtures/make_fetch_repos.sh | 3 ++- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/git-repository/src/remote/connection/fetch/update_refs/tests.rs b/git-repository/src/remote/connection/fetch/update_refs/tests.rs index e25979e2e6a..551fb68a2e8 100644 --- a/git-repository/src/remote/connection/fetch/update_refs/tests.rs +++ b/git-repository/src/remote/connection/fetch/update_refs/tests.rs @@ -21,32 +21,40 @@ mod update { use git_ref::TargetRef; #[test] - #[ignore] fn various_valid_updates() { let repo = repo("two-origins"); // TODO: test reflog message (various cases if it's new) - for (spec, expected_mode, detail) in [ + for (spec, expected_mode, has_edit_index, detail) in [ ( "refs/heads/main:refs/remotes/origin/main", fetch::refs::update::Mode::NoChangeNeeded, + true, "these refs are en-par since the initial clone", ), ( "refs/heads/main", fetch::refs::update::Mode::NoChangeNeeded, + false, "without local destination ref there is nothing to do for us, ever (except for FETCH_HEADs) later", ), ( "refs/heads/main:refs/remotes/origin/new-main", fetch::refs::update::Mode::New, + true, "the destination branch doesn't exist and needs to be created", ), ( - "+refs/heads/main:refs/heads/g", + "+refs/heads/main:refs/remotes/origin/g", fetch::refs::update::Mode::Forced, + true, "a forced non-fastforward (main goes backwards)", ), - // ("refs/heads/g:refs/heads/main", fetch::refs::update::Mode::FastForward, "a fast-forward only fast-forward situation, all good"), + // ( + // "refs/remotes/origin/g:refs/heads/not-currently-checked-out", + // fetch::refs::update::Mode::FastForward, + // true, + // "a fast-forward only fast-forward situation, all good", + // ), ] { let (mapping, specs) = mapping_from_spec(spec, &repo); let out = fetch::refs::update(&repo, &mapping, &specs, fetch::DryRun::Yes).unwrap(); @@ -55,12 +63,12 @@ mod update { out.updates, vec![fetch::refs::Update { mode: expected_mode, - edit_index: Some(0), + edit_index: has_edit_index.then(|| 0), spec_index: 0 }], "{spec:?}: {detail}" ); - assert_eq!(out.edits.len(), 1); + assert_eq!(out.edits.len(), has_edit_index.then(|| 1).unwrap_or(0)); } } diff --git a/git-repository/tests/fixtures/make_fetch_repos.sh b/git-repository/tests/fixtures/make_fetch_repos.sh index cecc027d442..07ac986b3c4 100644 --- a/git-repository/tests/fixtures/make_fetch_repos.sh +++ b/git-repository/tests/fixtures/make_fetch_repos.sh @@ -10,7 +10,8 @@ git clone --shared base clone-as-base-with-changes git tag -m "new-file introduction" v1.0 ) -git clone --bare --shared base two-origins +git clone --shared base two-origins (cd two-origins git remote add changes-on-top-of-origin "$PWD/../clone-as-base-with-changes" + git branch "not-currently-checked-out" ) From 7ced2402eb28301adc5330f336ece5eaf3bd9222 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 27 Sep 2022 07:55:38 +0800 Subject: [PATCH 080/113] tests for all the cases excluding fast-forwards (#450) --- .../connection/fetch/update_refs/tests.rs | 33 +++++++++++++++---- .../tests/fixtures/make_fetch_repos.sh | 1 + 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/git-repository/src/remote/connection/fetch/update_refs/tests.rs b/git-repository/src/remote/connection/fetch/update_refs/tests.rs index 551fb68a2e8..925a3ea7c3f 100644 --- a/git-repository/src/remote/connection/fetch/update_refs/tests.rs +++ b/git-repository/src/remote/connection/fetch/update_refs/tests.rs @@ -43,13 +43,19 @@ mod update { true, "the destination branch doesn't exist and needs to be created", ), + ( + "refs/heads/main:refs/remotes/origin/new-main", + fetch::refs::update::Mode::New, + true, + "just to validate that we really are in dry-run mode, or else this ref would be present now", + ), ( "+refs/heads/main:refs/remotes/origin/g", fetch::refs::update::Mode::Forced, true, "a forced non-fastforward (main goes backwards)", ), - // ( + // ( // TODO: make fast-forwards work // "refs/remotes/origin/g:refs/heads/not-currently-checked-out", // fetch::refs::update::Mode::FastForward, // true, @@ -73,18 +79,33 @@ mod update { } #[test] - #[ignore] + fn symbolic_refs_are_never_written() { + let repo = repo("two-origins"); + let (mappings, specs) = mapping_from_spec("refs/heads/main:refs/heads/symbolic", &repo); + let out = fetch::refs::update(&repo, &mappings, &specs, fetch::DryRun::Yes).unwrap(); + + assert_eq!( + out.updates, + vec![fetch::refs::Update { + mode: fetch::refs::update::Mode::RejectedSymbolic, + edit_index: None, + spec_index: 0, + }] + ); + assert_eq!(out.edits.len(), 0); + } + + #[test] #[should_panic] - fn fast_forward_is_not_implemented_yet() { - // TODO: move it above for acceptable case, test here for non-fastforwards being denied. + fn fast_forward_is_not_implemented_yet_but_should_be_denied() { let repo = repo("two-origins"); - let (mappings, specs) = mapping_from_spec("+refs/heads/main:refs/heads/g", &repo); + let (mappings, specs) = mapping_from_spec("refs/heads/main:refs/remotes/origin/g", &repo); let out = fetch::refs::update(&repo, &mappings, &specs, fetch::DryRun::Yes).unwrap(); assert_eq!( out.updates, vec![fetch::refs::Update { - mode: fetch::refs::update::Mode::FastForward, + mode: fetch::refs::update::Mode::RejectedNonFastForward, edit_index: Some(0), spec_index: 0, }] diff --git a/git-repository/tests/fixtures/make_fetch_repos.sh b/git-repository/tests/fixtures/make_fetch_repos.sh index 07ac986b3c4..934f9cc9681 100644 --- a/git-repository/tests/fixtures/make_fetch_repos.sh +++ b/git-repository/tests/fixtures/make_fetch_repos.sh @@ -14,4 +14,5 @@ git clone --shared base two-origins (cd two-origins git remote add changes-on-top-of-origin "$PWD/../clone-as-base-with-changes" git branch "not-currently-checked-out" + git symbolic-ref refs/heads/symbolic refs/heads/main ) From e25460b3519609b2836e5bf57ad59e6cf06872e4 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 27 Sep 2022 08:10:03 +0800 Subject: [PATCH 081/113] Add failing test to show we need to respect dry-run mode (or the lack thereof) (#450) --- .../connection/fetch/update_refs/update.rs | 16 +++++++++---- git-repository/tests/remote/fetch.rs | 24 ++++++++++++++++++- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/git-repository/src/remote/connection/fetch/update_refs/update.rs b/git-repository/src/remote/connection/fetch/update_refs/update.rs index eebe02f49a7..899f8e2a5f9 100644 --- a/git-repository/src/remote/connection/fetch/update_refs/update.rs +++ b/git-repository/src/remote/connection/fetch/update_refs/update.rs @@ -46,19 +46,25 @@ pub enum Mode { impl Outcome { /// Produce an iterator over all information used to produce the this outcome, ref-update by ref-update, using the `mappings` /// used when producing the ref update. - pub fn iter_mapping_updates<'a>( + pub fn iter_mapping_updates<'a, 'b>( &self, mappings: &'a [fetch::Mapping], + refspecs: &'b [git_refspec::RefSpec], ) -> impl Iterator< Item = ( &super::Update, &'a fetch::Mapping, + &'b git_refspec::RefSpec, Option<&git_ref::transaction::RefEdit>, ), > { - self.updates - .iter() - .zip(mappings.iter()) - .map(move |(update, mapping)| (update, mapping, update.edit_index.and_then(|idx| self.edits.get(idx)))) + self.updates.iter().zip(mappings.iter()).map(move |(update, mapping)| { + ( + update, + mapping, + &refspecs[mapping.spec_index], + update.edit_index.and_then(|idx| self.edits.get(idx)), + ) + }) } } diff --git a/git-repository/tests/remote/fetch.rs b/git-repository/tests/remote/fetch.rs index 6a04c9dc511..dd99e582305 100644 --- a/git-repository/tests/remote/fetch.rs +++ b/git-repository/tests/remote/fetch.rs @@ -22,6 +22,7 @@ mod blocking_io { } #[test] + #[ignore] fn fetch_pack() -> crate::Result { for version in [ None, @@ -64,7 +65,7 @@ mod blocking_io { match outcome.status { fetch::Status::Change { write_pack_bundle, - update_refs: _, // TODO: validate update refs + update_refs: refs, } => { assert_eq!(write_pack_bundle.pack_kind, git::odb::pack::data::Version::V2); assert_eq!(write_pack_bundle.object_hash, repo.object_hash()); @@ -79,6 +80,27 @@ mod blocking_io { ); assert!(write_pack_bundle.data_path.map_or(false, |f| f.is_file())); assert!(write_pack_bundle.index_path.map_or(false, |f| f.is_file())); + + assert_eq!( + refs.updates, + vec![fetch::refs::Update { + mode: fetch::refs::update::Mode::New, + edit_index: Some(0), + spec_index: 0 + }] + ); + for (_update, mapping, _spec, edit) in refs.iter_mapping_updates( + &outcome.ref_map.mappings, + remote.refspecs(git::remote::Direction::Fetch), + ) { + let edit = edit.expect("refedit present even if it's a no-op"); + let r = repo.find_reference(edit.name.as_ref()).unwrap(); + assert_eq!( + r.id(), + *mapping.remote.as_id(), + "local reference should point to remote id" + ); + } } fetch::Status::NoChange => unreachable!("we firmly expect changes here"), } From 7076891fb1b44cd442928f7e56f53f4b085e7a11 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 27 Sep 2022 08:17:23 +0800 Subject: [PATCH 082/113] add information about planned lock timeout support (from configuration) (#450) --- src/plumbing/progress.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/plumbing/progress.rs b/src/plumbing/progress.rs index b49de4c00be..8d04a2e142a 100644 --- a/src/plumbing/progress.rs +++ b/src/plumbing/progress.rs @@ -116,6 +116,14 @@ static GIT_CONFIG: &[Record] = &[ deviation: None, }, }, + Record { + config: "core.filesRefLockTimeout", + usage: Planned {note: Some("to be cached and used for all ref operations")}, + }, + Record { + config: "core.packedRefsTimeout", + usage: Planned {note: Some("needs support in git-ref crate which currently only knows one setting for all locks")}, + }, Record { config: "core.logAllRefUpdates", usage: InModule { From 8fe4bf4bcc463154c54082df5f38f0cd801915fb Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 27 Sep 2022 08:26:30 +0800 Subject: [PATCH 083/113] actually apply ref updates (#450) --- .../src/remote/connection/fetch/update_refs/mod.rs | 9 ++++++++- .../src/remote/connection/fetch/update_refs/tests.rs | 4 ++++ .../src/remote/connection/fetch/update_refs/update.rs | 2 ++ git-repository/src/remote/fetch.rs | 3 ++- git-repository/tests/remote/fetch.rs | 1 - 5 files changed, 16 insertions(+), 3 deletions(-) diff --git a/git-repository/src/remote/connection/fetch/update_refs/mod.rs b/git-repository/src/remote/connection/fetch/update_refs/mod.rs index 73fcdf96484..e6553a4bed2 100644 --- a/git-repository/src/remote/connection/fetch/update_refs/mod.rs +++ b/git-repository/src/remote/connection/fetch/update_refs/mod.rs @@ -26,7 +26,7 @@ pub(crate) fn update( repo: &crate::Repository, mappings: &[fetch::Mapping], refspecs: &[git_refspec::RefSpec], - _dry_run: fetch::DryRun, + dry_run: fetch::DryRun, ) -> Result { let mut edits = Vec::new(); let mut updates = Vec::new(); @@ -89,6 +89,13 @@ pub(crate) fn update( }) } + let edits = match dry_run { + fetch::DryRun::No => { + repo.edit_references(edits, git_lock::acquire::Fail::Immediately, repo.committer_or_default())? + } + fetch::DryRun::Yes => edits, + }; + Ok(update::Outcome { edits, updates }) } diff --git a/git-repository/src/remote/connection/fetch/update_refs/tests.rs b/git-repository/src/remote/connection/fetch/update_refs/tests.rs index 925a3ea7c3f..a897e3c9091 100644 --- a/git-repository/src/remote/connection/fetch/update_refs/tests.rs +++ b/git-repository/src/remote/connection/fetch/update_refs/tests.rs @@ -95,6 +95,10 @@ mod update { assert_eq!(out.edits.len(), 0); } + #[test] + #[ignore] + fn currently_checked_out_destination_is_rejected() {} + #[test] #[should_panic] fn fast_forward_is_not_implemented_yet_but_should_be_denied() { diff --git a/git-repository/src/remote/connection/fetch/update_refs/update.rs b/git-repository/src/remote/connection/fetch/update_refs/update.rs index 899f8e2a5f9..e8892e80596 100644 --- a/git-repository/src/remote/connection/fetch/update_refs/update.rs +++ b/git-repository/src/remote/connection/fetch/update_refs/update.rs @@ -9,6 +9,8 @@ mod error { FindReference(#[from] crate::reference::find::Error), #[error("A remote reference had a name that wasn't considered valid. Corrupt remote repo or insufficient checks on remote?")] InvalidRefName(#[from] git_validate::refname::Error), + #[error("Failed to update references to their new position to match their remote locations")] + EditReferences(#[from] crate::reference::edit::Error), } } diff --git a/git-repository/src/remote/fetch.rs b/git-repository/src/remote/fetch.rs index 7edb14f7683..588576ca74e 100644 --- a/git-repository/src/remote/fetch.rs +++ b/git-repository/src/remote/fetch.rs @@ -2,9 +2,10 @@ use crate::bstr::BString; /// If `Yes`, don't really make changes but do as much as possible to get an idea of what would be done. #[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[cfg_attr(not(test), allow(dead_code))] +#[cfg(feature = "blocking-network-client")] pub(crate) enum DryRun { /// Enable dry-run mode and don't actually change the underlying repository in any way. - #[cfg(test)] Yes, /// Run the operation like normal, making changes to the underlying repository. No, diff --git a/git-repository/tests/remote/fetch.rs b/git-repository/tests/remote/fetch.rs index dd99e582305..c1530f77f89 100644 --- a/git-repository/tests/remote/fetch.rs +++ b/git-repository/tests/remote/fetch.rs @@ -22,7 +22,6 @@ mod blocking_io { } #[test] - #[ignore] fn fetch_pack() -> crate::Result { for version in [ None, From 1bb910ee2dbe0c5f19aefd9669cebc305870953e Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 27 Sep 2022 10:29:26 +0800 Subject: [PATCH 084/113] =?UTF-8?q?prepare=20for=20worktree-aware=20checke?= =?UTF-8?q?d-out=20branch=20handling=E2=80=A6=20(#450)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …which seems to be a little more work than anticipated. Fortunately we already have worktrees implemented, but it seems to be a bit partial as one method is entirely 'sketched' that should be implemented now as it's what we need here. --- .../remote/connection/fetch/update_refs/tests.rs | 14 +++++++++----- .../remote/connection/fetch/update_refs/update.rs | 2 ++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/git-repository/src/remote/connection/fetch/update_refs/tests.rs b/git-repository/src/remote/connection/fetch/update_refs/tests.rs index a897e3c9091..ced23a2181d 100644 --- a/git-repository/src/remote/connection/fetch/update_refs/tests.rs +++ b/git-repository/src/remote/connection/fetch/update_refs/tests.rs @@ -21,6 +21,7 @@ mod update { use git_ref::TargetRef; #[test] + #[ignore] fn various_valid_updates() { let repo = repo("two-origins"); // TODO: test reflog message (various cases if it's new) @@ -55,6 +56,12 @@ mod update { true, "a forced non-fastforward (main goes backwards)", ), + ( + "+refs/remotes/origin/g:refs/heads/main", + fetch::refs::update::Mode::RejectedCheckedOut, + false, + "checked out branches cannot be written, as it requires a merge of sorts which isn't done here", + ), // ( // TODO: make fast-forwards work // "refs/remotes/origin/g:refs/heads/not-currently-checked-out", // fetch::refs::update::Mode::FastForward, @@ -90,15 +97,12 @@ mod update { mode: fetch::refs::update::Mode::RejectedSymbolic, edit_index: None, spec_index: 0, - }] + }], + "this also protects from writing HEAD, which should in theory be impossible to get from a refspec as it normalizes partial ref names" ); assert_eq!(out.edits.len(), 0); } - #[test] - #[ignore] - fn currently_checked_out_destination_is_rejected() {} - #[test] #[should_panic] fn fast_forward_is_not_implemented_yet_but_should_be_denied() { diff --git a/git-repository/src/remote/connection/fetch/update_refs/update.rs b/git-repository/src/remote/connection/fetch/update_refs/update.rs index e8892e80596..39952af57dd 100644 --- a/git-repository/src/remote/connection/fetch/update_refs/update.rs +++ b/git-repository/src/remote/connection/fetch/update_refs/update.rs @@ -40,6 +40,8 @@ pub enum Mode { RejectedNonFastForward, /// The update of a local symbolic reference was rejected. RejectedSymbolic, + /// The update was rejected as the destination branch is currently checked out + RejectedCheckedOut, /// No change was attempted as the remote ref didn't change compared to the current ref, or because no remote ref was specified /// in the ref-spec. NoChangeNeeded, From 3a0fb1b45c757add49677450836c0aaf6179a2b5 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 27 Sep 2022 11:23:06 +0800 Subject: [PATCH 085/113] change!: remote `lock_mode` from all methods dealing with reference edits. (#450) It is now read from `core.filesRefLockTimeout` accordingly. --- git-repository/src/config/cache/access.rs | 60 +++++++++++++++++++ git-repository/src/config/cache/init.rs | 1 + git-repository/src/config/cache/mod.rs | 23 +------ git-repository/src/config/mod.rs | 8 ++- git-repository/src/reference/edits.rs | 38 ++++-------- git-repository/src/reference/errors.rs | 2 + .../connection/fetch/update_refs/mod.rs | 4 +- git-repository/src/repository/object.rs | 1 - git-repository/src/repository/reference.rs | 16 ++--- git-repository/tests/repository/mod.rs | 2 +- src/plumbing/progress.rs | 2 +- 11 files changed, 90 insertions(+), 67 deletions(-) create mode 100644 git-repository/src/config/cache/access.rs diff --git a/git-repository/src/config/cache/access.rs b/git-repository/src/config/cache/access.rs new file mode 100644 index 00000000000..76f751df67a --- /dev/null +++ b/git-repository/src/config/cache/access.rs @@ -0,0 +1,60 @@ +use crate::config::Cache; +use crate::{remote, repository::identity}; +use git_lock::acquire::Fail; +use std::convert::TryInto; +use std::time::Duration; + +/// Access +impl Cache { + pub(crate) fn personas(&self) -> &identity::Personas { + self.personas + .get_or_init(|| identity::Personas::from_config_and_env(&self.resolved, self.git_prefix)) + } + + pub(crate) fn url_rewrite(&self) -> &remote::url::Rewrite { + self.url_rewrite + .get_or_init(|| remote::url::Rewrite::from_config(&self.resolved, self.filter_config_section)) + } + + #[cfg(any(feature = "blocking-network-client", feature = "async-network-client"))] + pub(crate) fn url_scheme( + &self, + ) -> Result<&remote::url::SchemePermission, remote::url::scheme_permission::init::Error> { + self.url_scheme.get_or_try_init(|| { + remote::url::SchemePermission::from_config(&self.resolved, self.git_prefix, self.filter_config_section) + }) + } + + /// Returns (file-timeout, pack-refs timeout) + pub(crate) fn lock_timeout( + &self, + ) -> Result<(git_lock::acquire::Fail, git_lock::acquire::Fail), git_config::value::Error> { + enum Kind { + RefFiles, + RefPackFile, + } + let mut out: [git_lock::acquire::Fail; 2] = Default::default(); + + for (idx, kind) in [Kind::RefFiles, Kind::RefPackFile].iter().enumerate() { + let (key, default_ms) = match kind { + Kind::RefFiles => ("filesRefLockTimeout", 100), + Kind::RefPackFile => ("packedRefsTimeout", 1000), + }; + let mk_default = || Fail::AfterDurationWithBackoff(Duration::from_millis(default_ms)); + let mut fnp = self.filter_config_section; + + let lock_mode = match self.resolved.integer_filter("core", None, key, &mut fnp) { + Some(Ok(val)) if val < 0 => Fail::AfterDurationWithBackoff(Duration::from_secs(u64::MAX)), + Some(Ok(val)) if val == 0 => Fail::Immediately, + Some(Ok(val)) => Fail::AfterDurationWithBackoff(Duration::from_millis( + val.try_into().expect("i64 can be repsented by u64"), + )), + Some(Err(_)) if self.lenient_config => mk_default(), + Some(Err(err)) => return Err(err), + None => mk_default(), + }; + out[idx] = lock_mode; + } + Ok((out[0], out[1])) + } +} diff --git a/git-repository/src/config/cache/init.rs b/git-repository/src/config/cache/init.rs index 15e9a3e4507..0b9ff7829c1 100644 --- a/git-repository/src/config/cache/init.rs +++ b/git-repository/src/config/cache/init.rs @@ -147,6 +147,7 @@ impl Cache { excludes_file, xdg_config_home_env, home_env, + lenient_config, personas: Default::default(), url_rewrite: Default::default(), #[cfg(any(feature = "blocking-network-client", feature = "async-network-client"))] diff --git a/git-repository/src/config/cache/mod.rs b/git-repository/src/config/cache/mod.rs index e8cf78ba300..ae42c218836 100644 --- a/git-repository/src/config/cache/mod.rs +++ b/git-repository/src/config/cache/mod.rs @@ -1,5 +1,4 @@ use super::{Cache, Error}; -use crate::{remote, repository::identity}; mod incubate; pub(crate) use incubate::StageOne; @@ -12,27 +11,7 @@ impl std::fmt::Debug for Cache { } } -/// Access -impl Cache { - pub(crate) fn personas(&self) -> &identity::Personas { - self.personas - .get_or_init(|| identity::Personas::from_config_and_env(&self.resolved, self.git_prefix)) - } - - pub(crate) fn url_rewrite(&self) -> &remote::url::Rewrite { - self.url_rewrite - .get_or_init(|| remote::url::Rewrite::from_config(&self.resolved, self.filter_config_section)) - } - - #[cfg(any(feature = "blocking-network-client", feature = "async-network-client"))] - pub(crate) fn url_scheme( - &self, - ) -> Result<&remote::url::SchemePermission, remote::url::scheme_permission::init::Error> { - self.url_scheme.get_or_try_init(|| { - remote::url::SchemePermission::from_config(&self.resolved, self.git_prefix, self.filter_config_section) - }) - } -} +mod access; mod util; pub(crate) use util::interpolate_context; diff --git a/git-repository/src/config/mod.rs b/git-repository/src/config/mod.rs index 49f6df8363d..1c9a9aa0c49 100644 --- a/git-repository/src/config/mod.rs +++ b/git-repository/src/config/mod.rs @@ -14,14 +14,15 @@ pub struct Snapshot<'repo> { pub(crate) repo: &'repo Repository, } -/// A platform to access configuration values and modify them in memory, while making them available when this platform is dropped. +/// A platform to access configuration values and modify them in memory, while making them available when this platform is dropped +/// as form of auto-commit. /// Note that the values will only affect this instance of the parent repository, and not other clones that may exist. /// /// Note that these values won't update even if the underlying file(s) change. /// /// Use [`forget()`][Self::forget()] to not apply any of the changes. // TODO: make it possible to load snapshots with reloading via .config() and write mutated snapshots back to disk which should be the way -// to affect all instances of a repo. +// to affect all instances of a repo, probably via `config_mut()` and `config_mut_at()`. pub struct SnapshotMut<'repo> { pub(crate) repo: &'repo mut Repository, pub(crate) config: git_config::File<'static>, @@ -86,6 +87,9 @@ pub(crate) struct Cache { pub ignore_case: bool, /// The path to the user-level excludes file to ignore certain files in the worktree. pub excludes_file: Option, + /// If true, we should default what's possible if something is misconfigured, on case by case basis, to be more resilient. + /// Also available in options! Keep in sync! + pub lenient_config: bool, /// Define how we can use values obtained with `xdg_config(…)` and its `XDG_CONFIG_HOME` variable. xdg_config_home_env: git_sec::Permission, /// Define how we can use values obtained with `xdg_config(…)`. and its `HOME` variable. diff --git a/git-repository/src/reference/edits.rs b/git-repository/src/reference/edits.rs index 3e39822af51..d3b43aff681 100644 --- a/git-repository/src/reference/edits.rs +++ b/git-repository/src/reference/edits.rs @@ -55,35 +55,23 @@ pub mod delete { use crate::Reference; - mod error { - /// The error returned by [`Reference::delete()`][super::Reference::delete()]. - #[derive(Debug, thiserror::Error)] - #[allow(missing_docs)] - pub enum Error { - #[error(transparent)] - ReferenceEdit(#[from] crate::reference::edit::Error), - } - } - pub use error::Error; - use git_lock::acquire::Fail; - impl<'repo> Reference<'repo> { /// Delete this reference or fail if it was changed since last observed. /// Note that this instance remains available in memory but probably shouldn't be used anymore. - pub fn delete(&self) -> Result<(), Error> { - self.repo.edit_reference( - RefEdit { - change: Change::Delete { - expected: PreviousValue::MustExistAndMatch(self.inner.target.clone()), - log: RefLog::AndReference, + pub fn delete(&self) -> Result<(), crate::reference::edit::Error> { + self.repo + .edit_reference( + RefEdit { + change: Change::Delete { + expected: PreviousValue::MustExistAndMatch(self.inner.target.clone()), + log: RefLog::AndReference, + }, + name: self.inner.name.clone(), + deref: false, }, - name: self.inner.name.clone(), - deref: false, - }, - Fail::Immediately, - self.repo.committer_or_default(), - )?; - Ok(()) + self.repo.committer_or_default(), + ) + .map(|_| ()) } } } diff --git a/git-repository/src/reference/errors.rs b/git-repository/src/reference/errors.rs index 15358ff0010..9854f8851bf 100644 --- a/git-repository/src/reference/errors.rs +++ b/git-repository/src/reference/errors.rs @@ -11,6 +11,8 @@ pub mod edit { FileTransactionCommit(#[from] git_ref::file::transaction::commit::Error), #[error(transparent)] NameValidation(#[from] git_validate::reference::name::Error), + #[error("Could not interpret core.filesRefLockTimeout or core.packedRefsTimeout, it must be the number in milliseconds to wait for locks or negative to wait forever")] + LockTimeoutConfiguration(#[from] git_config::value::Error), } } diff --git a/git-repository/src/remote/connection/fetch/update_refs/mod.rs b/git-repository/src/remote/connection/fetch/update_refs/mod.rs index e6553a4bed2..f94e3065907 100644 --- a/git-repository/src/remote/connection/fetch/update_refs/mod.rs +++ b/git-repository/src/remote/connection/fetch/update_refs/mod.rs @@ -90,9 +90,7 @@ pub(crate) fn update( } let edits = match dry_run { - fetch::DryRun::No => { - repo.edit_references(edits, git_lock::acquire::Fail::Immediately, repo.committer_or_default())? - } + fetch::DryRun::No => repo.edit_references(edits, repo.committer_or_default())?, fetch::DryRun::Yes => edits, }; diff --git a/git-repository/src/repository/object.rs b/git-repository/src/repository/object.rs index bfef691fb16..2fc40b12519 100644 --- a/git-repository/src/repository/object.rs +++ b/git-repository/src/repository/object.rs @@ -195,7 +195,6 @@ impl crate::Repository { name: reference, deref: true, }, - git_lock::acquire::Fail::Immediately, commit.committer.to_ref(), )?; Ok(commit_id) diff --git a/git-repository/src/repository/reference.rs b/git-repository/src/repository/reference.rs index eab411da703..af9e9b35c38 100644 --- a/git-repository/src/repository/reference.rs +++ b/git-repository/src/repository/reference.rs @@ -2,7 +2,6 @@ use std::convert::TryInto; use git_actor as actor; use git_hash::ObjectId; -use git_lock as lock; use git_ref::{ transaction::{Change, LogChange, PreviousValue, RefEdit, RefLog}, FullName, PartialNameRef, Target, @@ -10,8 +9,6 @@ use git_ref::{ use crate::{bstr::BString, ext::ReferenceExt, reference, Reference}; -const DEFAULT_LOCK_MODE: git_lock::acquire::Fail = git_lock::acquire::Fail::Immediately; - /// Obtain and alter references comfortably impl crate::Repository { /// Create a lightweight tag with given `name` (and without `refs/tags/` prefix) pointing to the given `target`, and return it as reference. @@ -35,7 +32,6 @@ impl crate::Repository { name: format!("refs/tags/{}", name.as_ref()).try_into()?, deref: false, }, - DEFAULT_LOCK_MODE, self.committer_or_default(), )?; assert_eq!(edits.len(), 1, "reference splits should ever happen"); @@ -109,7 +105,6 @@ impl crate::Repository { name, deref: false, }, - DEFAULT_LOCK_MODE, self.committer_or_default(), )?; assert_eq!( @@ -126,33 +121,30 @@ impl crate::Repository { .attach(self)) } - /// Edit a single reference as described in `edit`, handle locks via `lock_mode` and write reference logs as `log_committer`. + /// Edit a single reference as described in `edit`, and write reference logs as `log_committer`. /// /// One or more `RefEdit`s are returned - symbolic reference splits can cause more edits to be performed. All edits have the previous /// reference values set to the ones encountered at rest after acquiring the respective reference's lock. pub fn edit_reference( &self, edit: RefEdit, - lock_mode: lock::acquire::Fail, log_committer: actor::SignatureRef<'_>, ) -> Result, reference::edit::Error> { - self.edit_references(Some(edit), lock_mode, log_committer) + self.edit_references(Some(edit), log_committer) } - /// Edit one or more references as described by their `edits`, with `lock_mode` deciding on how to handle competing - /// transactions. `log_committer` is the name appearing in reference logs. + /// Edit one or more references as described by their `edits`. `log_committer` is the name appearing in reference logs. /// /// Returns all reference edits, which might be more than where provided due the splitting of symbolic references, and /// whose previous (_old_) values are the ones seen on in storage after the reference was locked. pub fn edit_references( &self, edits: impl IntoIterator, - lock_mode: lock::acquire::Fail, log_committer: actor::SignatureRef<'_>, ) -> Result, reference::edit::Error> { self.refs .transaction() - .prepare(edits, lock_mode)? + .prepare(edits, self.config.lock_timeout()?.0)? .commit(log_committer) .map_err(Into::into) } diff --git a/git-repository/tests/repository/mod.rs b/git-repository/tests/repository/mod.rs index 06ae361398e..6102a04cd2c 100644 --- a/git-repository/tests/repository/mod.rs +++ b/git-repository/tests/repository/mod.rs @@ -10,7 +10,7 @@ mod worktree; #[test] fn size_in_memory() { - let expected = [760, 800]; + let expected = [768, 808]; let actual_size = std::mem::size_of::(); assert!( expected.contains(&actual_size), diff --git a/src/plumbing/progress.rs b/src/plumbing/progress.rs index 8d04a2e142a..d7ddc9ebf21 100644 --- a/src/plumbing/progress.rs +++ b/src/plumbing/progress.rs @@ -118,7 +118,7 @@ static GIT_CONFIG: &[Record] = &[ }, Record { config: "core.filesRefLockTimeout", - usage: Planned {note: Some("to be cached and used for all ref operations")}, + usage: InModule {name: "config::cache::access", deviation: None}, }, Record { config: "core.packedRefsTimeout", From e88de0f948325773db1925b07aa878e1dbb76bad Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 27 Sep 2022 16:06:35 +0800 Subject: [PATCH 086/113] change!: All methods editing references don't take the author as parameter anymore. (#450) Instead, these are taken from the git configuration and can be configured on the fly with temporarily altered configuration. --- .../examples/init-repo-and-commit.rs | 65 ++++++++----------- git-repository/src/config/mod.rs | 8 ++- git-repository/src/config/snapshot/access.rs | 18 +++-- git-repository/src/config/snapshot/mod.rs | 27 +++++++- git-repository/src/reference/edits.rs | 17 ++--- .../connection/fetch/update_refs/mod.rs | 2 +- git-repository/src/repository/config.rs | 5 +- git-repository/src/repository/identity.rs | 11 +++- git-repository/src/repository/object.rs | 53 +++++++-------- git-repository/src/repository/reference.rs | 60 +++++++---------- git-repository/tests/repository/object.rs | 42 +++--------- git-repository/tests/util/mod.rs | 12 +++- 12 files changed, 163 insertions(+), 157 deletions(-) diff --git a/git-repository/examples/init-repo-and-commit.rs b/git-repository/examples/init-repo-and-commit.rs index 796bd04b8ff..bcf237d2729 100644 --- a/git-repository/examples/init-repo-and-commit.rs +++ b/git-repository/examples/init-repo-and-commit.rs @@ -13,49 +13,36 @@ fn main() -> anyhow::Result<()> { let git_dir = std::env::args_os() .nth(1) .context("First argument needs to be the directory to initialize the repository in")?; - let repo = git::init_bare(git_dir)?; + let mut repo = git::init_bare(git_dir)?; println!("Repo (bare): {:?}", repo.git_dir()); let mut tree = git::objs::Tree::empty(); - let empty_tree_id = repo.write_object(&tree)?; - - let author = git::actor::SignatureRef { - name: "Maria Sanchez".into(), - email: "maria@example.com".into(), - time: git_date::Time::now_local_or_utc(), - }; - let initial_commit_id = repo.commit( - "HEAD", - author, - author, - "initial commit", - empty_tree_id, - git::commit::NO_PARENT_IDS, - )?; - - println!("initial commit id with empty tree: {:?}", initial_commit_id); - - let blob_id = repo.write_blob("hello world")?.into(); - let entry = tree::Entry { - mode: tree::EntryMode::Blob, - oid: blob_id, - filename: "hello.txt".into(), - }; - - tree.entries.push(entry); - let hello_tree_id = repo.write_object(&tree)?; - - let blob_commit_id = repo.commit( - "HEAD", - author, - author, - "hello commit", - hello_tree_id, - [initial_commit_id], - )?; - - println!("commit id for 'hello world' blob: {:?}", blob_commit_id); + let empty_tree_id = repo.write_object(&tree)?.detach(); + + let mut config = repo.config_snapshot_mut(); + config.set_raw_value("author", None, "name", "Maria Sanchez")?; + config.set_raw_value("author", None, "email", "maria@example.com")?; + { + let repo = config.commit_and_rollback(); + let initial_commit_id = repo.commit("HEAD", "initial commit", empty_tree_id, git::commit::NO_PARENT_IDS)?; + + println!("initial commit id with empty tree: {:?}", initial_commit_id); + + let blob_id = repo.write_blob("hello world")?.into(); + let entry = tree::Entry { + mode: tree::EntryMode::Blob, + oid: blob_id, + filename: "hello.txt".into(), + }; + + tree.entries.push(entry); + let hello_tree_id = repo.write_object(&tree)?; + + let blob_commit_id = repo.commit("HEAD", "hello commit", hello_tree_id, [initial_commit_id])?; + + println!("commit id for 'hello world' blob: {:?}", blob_commit_id); + } Ok(()) } diff --git a/git-repository/src/config/mod.rs b/git-repository/src/config/mod.rs index 1c9a9aa0c49..f593e19218e 100644 --- a/git-repository/src/config/mod.rs +++ b/git-repository/src/config/mod.rs @@ -24,10 +24,16 @@ pub struct Snapshot<'repo> { // TODO: make it possible to load snapshots with reloading via .config() and write mutated snapshots back to disk which should be the way // to affect all instances of a repo, probably via `config_mut()` and `config_mut_at()`. pub struct SnapshotMut<'repo> { - pub(crate) repo: &'repo mut Repository, + pub(crate) repo: Option<&'repo mut Repository>, pub(crate) config: git_config::File<'static>, } +/// A utility structure created by [`SnapshotMut::commit_and_rollback()`] that restores the previous configuration on drop. +pub struct CommitAndRollback<'repo> { + pub(crate) repo: &'repo mut Repository, + pub(crate) prev_config: crate::Config, +} + pub(crate) mod section { pub fn is_trusted(meta: &git_config::file::Metadata) -> bool { meta.trust == git_sec::Trust::Full || meta.source.kind() != git_config::source::Kind::Repository diff --git a/git-repository/src/config/snapshot/access.rs b/git-repository/src/config/snapshot/access.rs index b9cf25d5db8..7ccd5b25f4c 100644 --- a/git-repository/src/config/snapshot/access.rs +++ b/git-repository/src/config/snapshot/access.rs @@ -1,6 +1,7 @@ +use git_features::threading::OwnShared; use std::borrow::Cow; -use crate::config::SnapshotMut; +use crate::config::{CommitAndRollback, SnapshotMut}; use crate::{ bstr::BStr, config::{cache::interpolate_context, Snapshot}, @@ -99,8 +100,17 @@ impl<'repo> SnapshotMut<'repo> { /// Note that this would also happen once this instance is dropped, but using this method may be more intuitive. pub fn commit(self) {} - /// Don't apply any of the changes after consuming this instance, effectively forgetting them. - pub fn forget(mut self) { - std::mem::take(&mut self.config); + /// Create a structure the temporarily commits the changes, but rolls them back when dropped. + pub fn commit_and_rollback(mut self) -> CommitAndRollback<'repo> { + let new_config = std::mem::take(&mut self.config); + let repo = self.repo.take().expect("this only runs once on consumption"); + repo.config.resolved = new_config.into(); + let prev_config = OwnShared::clone(&repo.config.resolved); + CommitAndRollback { repo, prev_config } + } + + /// Don't apply any of the changes after consuming this instance, effectively forgetting them, returning the changed configuration. + pub fn forget(mut self) -> git_config::File<'static> { + std::mem::take(&mut self.config) } } diff --git a/git-repository/src/config/snapshot/mod.rs b/git-repository/src/config/snapshot/mod.rs index 6fc03e002e1..88b4ab6fb33 100644 --- a/git-repository/src/config/snapshot/mod.rs +++ b/git-repository/src/config/snapshot/mod.rs @@ -7,12 +7,13 @@ pub mod apply_cli_overrides; pub mod credential_helpers; mod _impls { + use git_features::threading::OwnShared; use std::{ fmt::{Debug, Formatter}, ops::{Deref, DerefMut}, }; - use crate::config::{Snapshot, SnapshotMut}; + use crate::config::{CommitAndRollback, Snapshot, SnapshotMut}; impl Debug for Snapshot<'_> { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { @@ -20,6 +21,12 @@ mod _impls { } } + impl Debug for CommitAndRollback<'_> { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.repo.config.resolved.to_string()) + } + } + impl Debug for SnapshotMut<'_> { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { f.write_str(&self.config.to_string()) @@ -28,7 +35,15 @@ mod _impls { impl Drop for SnapshotMut<'_> { fn drop(&mut self) { - self.repo.config.resolved = std::mem::take(&mut self.config).into(); + if let Some(repo) = self.repo.take() { + repo.config.resolved = std::mem::take(&mut self.config).into() + }; + } + } + + impl Drop for CommitAndRollback<'_> { + fn drop(&mut self) { + self.repo.config.resolved = OwnShared::clone(&self.prev_config); } } @@ -40,6 +55,14 @@ mod _impls { } } + impl Deref for CommitAndRollback<'_> { + type Target = crate::Repository; + + fn deref(&self) -> &Self::Target { + self.repo + } + } + impl DerefMut for SnapshotMut<'_> { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.config diff --git a/git-repository/src/reference/edits.rs b/git-repository/src/reference/edits.rs index d3b43aff681..7f967b7f127 100644 --- a/git-repository/src/reference/edits.rs +++ b/git-repository/src/reference/edits.rs @@ -60,17 +60,14 @@ pub mod delete { /// Note that this instance remains available in memory but probably shouldn't be used anymore. pub fn delete(&self) -> Result<(), crate::reference::edit::Error> { self.repo - .edit_reference( - RefEdit { - change: Change::Delete { - expected: PreviousValue::MustExistAndMatch(self.inner.target.clone()), - log: RefLog::AndReference, - }, - name: self.inner.name.clone(), - deref: false, + .edit_reference(RefEdit { + change: Change::Delete { + expected: PreviousValue::MustExistAndMatch(self.inner.target.clone()), + log: RefLog::AndReference, }, - self.repo.committer_or_default(), - ) + name: self.inner.name.clone(), + deref: false, + }) .map(|_| ()) } } diff --git a/git-repository/src/remote/connection/fetch/update_refs/mod.rs b/git-repository/src/remote/connection/fetch/update_refs/mod.rs index f94e3065907..44488e93e6d 100644 --- a/git-repository/src/remote/connection/fetch/update_refs/mod.rs +++ b/git-repository/src/remote/connection/fetch/update_refs/mod.rs @@ -90,7 +90,7 @@ pub(crate) fn update( } let edits = match dry_run { - fetch::DryRun::No => repo.edit_references(edits, repo.committer_or_default())?, + fetch::DryRun::No => repo.edit_references(edits)?, fetch::DryRun::Yes => edits, }; diff --git a/git-repository/src/repository/config.rs b/git-repository/src/repository/config.rs index 860136bc969..3a2de9298ea 100644 --- a/git-repository/src/repository/config.rs +++ b/git-repository/src/repository/config.rs @@ -16,7 +16,10 @@ impl crate::Repository { /// of the [`Repository`][crate::Repository]. pub fn config_snapshot_mut(&mut self) -> config::SnapshotMut<'_> { let config = self.config.resolved.as_ref().clone(); - config::SnapshotMut { repo: self, config } + config::SnapshotMut { + repo: Some(self), + config, + } } /// The options used to open the repository. diff --git a/git-repository/src/repository/identity.rs b/git-repository/src/repository/identity.rs index 98759a13de2..18bba3061a9 100644 --- a/git-repository/src/repository/identity.rs +++ b/git-repository/src/repository/identity.rs @@ -4,7 +4,8 @@ use crate::bstr::BString; /// Identity handling. impl crate::Repository { - /// Return a crate-specific constant signature with [`Time`][git_actor::Time] set to now, + /// Return a crate-specific constant signature with [`Time`][git_actor::Time] set to now, or whatever + /// was overridden via `GIT_COMMITTER_TIME` or `GIT_AUTHOR_TIME` if these variables are allowed to be read, /// in a similar vein as the default that git chooses if there is nothing configured. /// /// This can be useful as fallback for an unset `committer` or `author`. @@ -16,7 +17,13 @@ impl crate::Repository { git_actor::SignatureRef { name: "gitoxide".into(), email: "gitoxide@localhost".into(), - time: git_date::Time::now_local_or_utc(), + time: { + let p = self.config.personas(); + p.committer + .time + .or(p.author.time) + .unwrap_or_else(git_date::Time::now_local_or_utc) + }, } } diff --git a/git-repository/src/repository/object.rs b/git-repository/src/repository/object.rs index 2fc40b12519..26c1260e9be 100644 --- a/git-repository/src/repository/object.rs +++ b/git-repository/src/repository/object.rs @@ -125,8 +125,10 @@ impl crate::Repository { self.tag_reference(name, tag_id, constraint).map_err(Into::into) } - /// Create a new commit object with `author`, `committer` and `message` referring to `tree` with `parents`, and point `reference` + /// Create a new commit object with `message` referring to `tree` with `parents`, and point `reference` /// to it. The commit is written without message encoding field, which can be assumed to be UTF-8. + /// `author` and `committer` fields are pre-set from the configuration, which can be altered + /// [temporarily][crate::Repository::config_snapshot_mut()] before the call if required. /// /// `reference` will be created if it doesn't exist, and can be `"HEAD"` to automatically write-through to the symbolic reference /// that `HEAD` points to if it is not detached. For this reason, detached head states cannot be created unless the `HEAD` is detached @@ -139,8 +141,6 @@ impl crate::Repository { pub fn commit( &self, reference: Name, - author: git_actor::SignatureRef<'_>, - committer: git_actor::SignatureRef<'_>, message: impl AsRef, tree: impl Into, parents: impl IntoIterator>, @@ -157,6 +157,8 @@ impl crate::Repository { // TODO: possibly use CommitRef to save a few allocations (but will have to allocate for object ids anyway. // This can be made vastly more efficient though if we wanted to, so we lie in the API let reference = reference.try_into()?; + let author = self.author_or_default(); + let committer = self.committer_or_default(); let commit = git_object::Commit { message: message.as_ref().into(), tree: tree.into(), @@ -168,35 +170,28 @@ impl crate::Repository { }; let commit_id = self.write_object(&commit)?; - self.edit_reference( - RefEdit { - change: Change::Update { - log: LogChange { - mode: RefLog::AndReference, - force_create_reflog: false, - message: crate::reference::log::message( - "commit", - commit.message.as_ref(), - commit.parents.len(), - ), - }, - expected: match commit.parents.first().map(|p| Target::Peeled(*p)) { - Some(previous) => { - if reference.as_bstr() == "HEAD" { - PreviousValue::MustExistAndMatch(previous) - } else { - PreviousValue::ExistingMustMatch(previous) - } + self.edit_reference(RefEdit { + change: Change::Update { + log: LogChange { + mode: RefLog::AndReference, + force_create_reflog: false, + message: crate::reference::log::message("commit", commit.message.as_ref(), commit.parents.len()), + }, + expected: match commit.parents.first().map(|p| Target::Peeled(*p)) { + Some(previous) => { + if reference.as_bstr() == "HEAD" { + PreviousValue::MustExistAndMatch(previous) + } else { + PreviousValue::ExistingMustMatch(previous) } - None => PreviousValue::MustNotExist, - }, - new: Target::Peeled(commit_id.inner), + } + None => PreviousValue::MustNotExist, }, - name: reference, - deref: true, + new: Target::Peeled(commit_id.inner), }, - commit.committer.to_ref(), - )?; + name: reference, + deref: true, + })?; Ok(commit_id) } } diff --git a/git-repository/src/repository/reference.rs b/git-repository/src/repository/reference.rs index af9e9b35c38..32e3288cb42 100644 --- a/git-repository/src/repository/reference.rs +++ b/git-repository/src/repository/reference.rs @@ -1,6 +1,5 @@ use std::convert::TryInto; -use git_actor as actor; use git_hash::ObjectId; use git_ref::{ transaction::{Change, LogChange, PreviousValue, RefEdit, RefLog}, @@ -22,18 +21,15 @@ impl crate::Repository { constraint: PreviousValue, ) -> Result, reference::edit::Error> { let id = target.into(); - let mut edits = self.edit_reference( - RefEdit { - change: Change::Update { - log: Default::default(), - expected: constraint, - new: Target::Peeled(id), - }, - name: format!("refs/tags/{}", name.as_ref()).try_into()?, - deref: false, + let mut edits = self.edit_reference(RefEdit { + change: Change::Update { + log: Default::default(), + expected: constraint, + new: Target::Peeled(id), }, - self.committer_or_default(), - )?; + name: format!("refs/tags/{}", name.as_ref()).try_into()?, + deref: false, + })?; assert_eq!(edits.len(), 1, "reference splits should ever happen"); let edit = edits.pop().expect("exactly one item"); Ok(Reference { @@ -91,22 +87,19 @@ impl crate::Repository { { let name = name.try_into().map_err(git_validate::reference::name::Error::from)?; let id = target.into(); - let mut edits = self.edit_reference( - RefEdit { - change: Change::Update { - log: LogChange { - mode: RefLog::AndReference, - force_create_reflog: false, - message: log_message.into(), - }, - expected: constraint, - new: Target::Peeled(id), + let mut edits = self.edit_reference(RefEdit { + change: Change::Update { + log: LogChange { + mode: RefLog::AndReference, + force_create_reflog: false, + message: log_message.into(), }, - name, - deref: false, + expected: constraint, + new: Target::Peeled(id), }, - self.committer_or_default(), - )?; + name, + deref: false, + })?; assert_eq!( edits.len(), 1, @@ -125,27 +118,24 @@ impl crate::Repository { /// /// One or more `RefEdit`s are returned - symbolic reference splits can cause more edits to be performed. All edits have the previous /// reference values set to the ones encountered at rest after acquiring the respective reference's lock. - pub fn edit_reference( - &self, - edit: RefEdit, - log_committer: actor::SignatureRef<'_>, - ) -> Result, reference::edit::Error> { - self.edit_references(Some(edit), log_committer) + pub fn edit_reference(&self, edit: RefEdit) -> Result, reference::edit::Error> { + self.edit_references(Some(edit)) } - /// Edit one or more references as described by their `edits`. `log_committer` is the name appearing in reference logs. + /// Edit one or more references as described by their `edits`. + /// Note that one can set the committer name for use in the ref-log by temporarily + /// [overriding the git-config][crate::Repository::config_snapshot_mut()]. /// /// Returns all reference edits, which might be more than where provided due the splitting of symbolic references, and /// whose previous (_old_) values are the ones seen on in storage after the reference was locked. pub fn edit_references( &self, edits: impl IntoIterator, - log_committer: actor::SignatureRef<'_>, ) -> Result, reference::edit::Error> { self.refs .transaction() .prepare(edits, self.config.lock_timeout()?.0)? - .commit(log_committer) + .commit(self.committer_or_default()) .map_err(Into::into) } diff --git a/git-repository/tests/repository/object.rs b/git-repository/tests/repository/object.rs index 27d05821f1b..113177f44b2 100644 --- a/git-repository/tests/repository/object.rs +++ b/git-repository/tests/repository/object.rs @@ -147,23 +147,17 @@ mod tag { } mod commit { + use crate::freeze_time; use git_repository as git; use git_testtools::hex_to_id; + #[test] fn parent_in_initial_commit_causes_failure() { let tmp = tempfile::tempdir().unwrap(); let repo = git::init(&tmp).unwrap(); let empty_tree_id = repo.write_object(&git::objs::Tree::empty()).unwrap().detach(); - let author = git::actor::Signature::empty(); let err = repo - .commit( - "HEAD", - author.to_ref(), - author.to_ref(), - "initial", - empty_tree_id, - [empty_tree_id], - ) + .commit("HEAD", "initial", empty_tree_id, [empty_tree_id]) .unwrap_err(); assert_eq!( err.to_string(), @@ -174,21 +168,14 @@ mod commit { #[test] fn single_line_initial_commit_empty_tree_ref_nonexisting() -> crate::Result { + freeze_time(); let tmp = tempfile::tempdir()?; let repo = git::init(&tmp)?; let empty_tree_id = repo.write_object(&git::objs::Tree::empty())?; - let author = git::actor::Signature::empty(); - let commit_id = repo.commit( - "HEAD", - author.to_ref(), - author.to_ref(), - "initial", - empty_tree_id, - git::commit::NO_PARENT_IDS, - )?; + let commit_id = repo.commit("HEAD", "initial", empty_tree_id, git::commit::NO_PARENT_IDS)?; assert_eq!( commit_id, - hex_to_id("302ea5640358f98ba23cda66c1e664a6f274643f"), + hex_to_id("bbbd91e8b0f81f0693291d4a47c3f2c724ac44ee"), "the commit id is stable" ); @@ -208,6 +195,7 @@ mod commit { #[test] fn multi_line_commit_message_uses_first_line_in_ref_log_ref_nonexisting() -> crate::Result { + freeze_time(); let (repo, _keep) = crate::basic_rw_repo()?; let parent = repo.find_reference("HEAD")?.peel_to_id_in_place()?; let empty_tree_id = parent.object()?.to_commit_ref_iter().tree_id().expect("tree to be set"); @@ -221,18 +209,10 @@ mod commit { empty_tree_id, "try and non-try work the same" ); - let author = git::actor::Signature::empty(); - let first_commit_id = repo.commit( - "HEAD", - author.to_ref(), - author.to_ref(), - "hello there \r\n\nthe body", - empty_tree_id, - Some(parent), - )?; + let first_commit_id = repo.commit("HEAD", "hello there \r\n\nthe body", empty_tree_id, Some(parent))?; assert_eq!( first_commit_id, - hex_to_id("1ff7decccf76bfa15bfdb0b66bac0c9144b4b083"), + hex_to_id("e7c7273539cfc1a52802fa9d61aa578f6ccebcb4"), "the commit id is stable" ); @@ -254,8 +234,6 @@ mod commit { let second_commit_id = repo.commit( "refs/heads/new-branch", - author.to_ref(), - author.to_ref(), "committing into a new branch creates it", empty_tree_id, Some(first_commit_id), @@ -263,7 +241,7 @@ mod commit { assert_eq!( second_commit_id, - hex_to_id("b0d041ade77e51d31c79c7147fb769336ccc77b1"), + hex_to_id("e1412f169e0812eb260601bdab3854ca0f1a7b33"), "the second commit id is stable" ); diff --git a/git-repository/tests/util/mod.rs b/git-repository/tests/util/mod.rs index e2ed60a4f80..8710c7cd91d 100644 --- a/git-repository/tests/util/mod.rs +++ b/git-repository/tests/util/mod.rs @@ -1,7 +1,15 @@ use git_repository::{open, Repository, ThreadSafeRepository}; +use once_cell::sync::Lazy; pub type Result = std::result::Result>; +pub fn freeze_time() { + static FROZEN: Lazy<()> = Lazy::new(|| { + std::env::set_var("GIT_AUTHOR_DATE", "1979-02-26 18:30:00"); + std::env::set_var("GIT_COMMITTER_DATE", "1979-02-26 18:30:00"); + }); + *FROZEN +} pub fn repo(name: &str) -> Result { let repo_path = git_testtools::scripted_fixture_repo_read_only(name)?; Ok(ThreadSafeRepository::open_opts(repo_path, restricted())?) @@ -13,7 +21,9 @@ pub fn named_repo(name: &str) -> Result { } pub fn restricted() -> open::Options { - open::Options::isolated() + let mut opts = open::Options::isolated(); + opts.permissions.env.git_prefix = git_sec::Permission::Allow; + opts } pub fn repo_rw(name: &str) -> Result<(Repository, tempfile::TempDir)> { From da147bffd7538453221b08f9f68a1332cfa3ebe3 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 27 Sep 2022 19:17:50 +0800 Subject: [PATCH 087/113] auto-update previously cached values after changing the configuration (#450) --- git-repository/src/config/cache/access.rs | 26 ++++++++ git-repository/src/config/cache/init.rs | 58 ++++++++++------- git-repository/src/config/snapshot/_impls.rs | 63 +++++++++++++++++++ git-repository/src/config/snapshot/mod.rs | 65 +------------------- 4 files changed, 125 insertions(+), 87 deletions(-) create mode 100644 git-repository/src/config/snapshot/_impls.rs diff --git a/git-repository/src/config/cache/access.rs b/git-repository/src/config/cache/access.rs index 76f751df67a..67c7f8d6ea6 100644 --- a/git-repository/src/config/cache/access.rs +++ b/git-repository/src/config/cache/access.rs @@ -2,6 +2,7 @@ use crate::config::Cache; use crate::{remote, repository::identity}; use git_lock::acquire::Fail; use std::convert::TryInto; +use std::path::PathBuf; use std::time::Duration; /// Access @@ -57,4 +58,29 @@ impl Cache { } Ok((out[0], out[1])) } + + /// Return a path by using the `$XDF_CONFIG_HOME` or `$HOME/.config/…` environment variables locations. + pub fn xdg_config_path( + &self, + resource_file_name: &str, + ) -> Result, git_sec::permission::Error> { + std::env::var_os("XDG_CONFIG_HOME") + .map(|path| (path, &self.xdg_config_home_env)) + .or_else(|| std::env::var_os("HOME").map(|path| (path, &self.home_env))) + .and_then(|(base, permission)| { + let resource = std::path::PathBuf::from(base).join("git").join(resource_file_name); + permission.check(resource).transpose() + }) + .transpose() + } + + /// Return the home directory if we are allowed to read it and if it is set in the environment. + /// + /// We never fail for here even if the permission is set to deny as we `git-config` will fail later + /// if it actually wants to use the home directory - we don't want to fail prematurely. + pub fn home_dir(&self) -> Option { + std::env::var_os("HOME") + .map(PathBuf::from) + .and_then(|path| self.home_env.check_opt(path)) + } } diff --git a/git-repository/src/config/cache/init.rs b/git-repository/src/config/cache/init.rs index 0b9ff7829c1..3fbecadcaaa 100644 --- a/git-repository/src/config/cache/init.rs +++ b/git-repository/src/config/cache/init.rs @@ -1,5 +1,3 @@ -use std::path::PathBuf; - use super::{interpolate_context, util, Error, StageOne}; use crate::{config::Cache, repository}; @@ -134,6 +132,7 @@ impl Cache { let ignore_case = config_bool(&config, "core.ignoreCase", false, lenient_config)?; let use_multi_pack_index = config_bool(&config, "core.multiPackIndex", true, lenient_config)?; let object_kind_hint = util::disambiguate_hint(&config); + // NOTE: When adding a new initial cache, consider adjusting `reread_values_and_clear_caches()` as well. Ok(Cache { resolved: config.into(), use_multi_pack_index, @@ -156,28 +155,41 @@ impl Cache { }) } - /// Return a path by using the `$XDF_CONFIG_HOME` or `$HOME/.config/…` environment variables locations. - pub fn xdg_config_path( - &self, - resource_file_name: &str, - ) -> Result, git_sec::permission::Error> { - std::env::var_os("XDG_CONFIG_HOME") - .map(|path| (path, &self.xdg_config_home_env)) - .or_else(|| std::env::var_os("HOME").map(|path| (path, &self.home_env))) - .and_then(|(base, permission)| { - let resource = std::path::PathBuf::from(base).join("git").join(resource_file_name); - permission.check(resource).transpose() - }) + /// Call this after the `resolved` configuration changed in a way that may affect the caches provided here. + /// + /// Note that we unconditionally re-read all values. + pub fn reread_values_and_clear_caches(&mut self) -> Result<(), Error> { + let config = &self.resolved; + + let home = self.home_dir(); + let install_dir = crate::path::install_dir().ok(); + let ctx = interpolate_context(install_dir.as_deref(), home.as_deref()); + self.excludes_file = match config + .path_filter("core", None, "excludesFile", &mut self.filter_config_section) + .map(|p| p.interpolate(ctx).map(|p| p.into_owned())) .transpose() - } + { + Ok(f) => f, + Err(_err) if self.lenient_config => None, + Err(err) => return Err(err.into()), + }; - /// Return the home directory if we are allowed to read it and if it is set in the environment. - /// - /// We never fail for here even if the permission is set to deny as we `git-config` will fail later - /// if it actually wants to use the home directory - we don't want to fail prematurely. - pub fn home_dir(&self) -> Option { - std::env::var_os("HOME") - .map(PathBuf::from) - .and_then(|path| self.home_env.check_opt(path)) + self.hex_len = match util::parse_core_abbrev(&config, self.object_hash) { + Ok(v) => v, + Err(_err) if self.lenient_config => None, + Err(err) => return Err(err), + }; + + use util::config_bool; + self.ignore_case = config_bool(&config, "core.ignoreCase", false, self.lenient_config)?; + self.object_kind_hint = util::disambiguate_hint(&config); + self.personas = Default::default(); + self.url_rewrite = Default::default(); + #[cfg(any(feature = "blocking-network-client", feature = "async-network-client"))] + { + self.url_scheme = Default::default(); + } + + Ok(()) } } diff --git a/git-repository/src/config/snapshot/_impls.rs b/git-repository/src/config/snapshot/_impls.rs new file mode 100644 index 00000000000..b449addf63d --- /dev/null +++ b/git-repository/src/config/snapshot/_impls.rs @@ -0,0 +1,63 @@ +use git_features::threading::OwnShared; +use std::{ + fmt::{Debug, Formatter}, + ops::{Deref, DerefMut}, +}; + +use crate::config::{CommitAndRollback, Snapshot, SnapshotMut}; + +impl Debug for Snapshot<'_> { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.repo.config.resolved.to_string()) + } +} + +impl Debug for CommitAndRollback<'_> { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.repo.config.resolved.to_string()) + } +} + +impl Debug for SnapshotMut<'_> { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.config.to_string()) + } +} + +impl Drop for SnapshotMut<'_> { + fn drop(&mut self) { + if let Some(repo) = self.repo.take() { + repo.config.resolved = std::mem::take(&mut self.config).into(); + repo.config.reread_values_and_clear_caches().ok(); + }; + } +} + +impl Drop for CommitAndRollback<'_> { + fn drop(&mut self) { + self.repo.config.resolved = OwnShared::clone(&self.prev_config); + self.repo.config.reread_values_and_clear_caches().ok(); + } +} + +impl Deref for SnapshotMut<'_> { + type Target = git_config::File<'static>; + + fn deref(&self) -> &Self::Target { + &self.config + } +} + +impl Deref for CommitAndRollback<'_> { + type Target = crate::Repository; + + fn deref(&self) -> &Self::Target { + self.repo + } +} + +impl DerefMut for SnapshotMut<'_> { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.config + } +} diff --git a/git-repository/src/config/snapshot/mod.rs b/git-repository/src/config/snapshot/mod.rs index 88b4ab6fb33..2f44ef851f9 100644 --- a/git-repository/src/config/snapshot/mod.rs +++ b/git-repository/src/config/snapshot/mod.rs @@ -1,3 +1,4 @@ +mod _impls; mod access; /// @@ -5,67 +6,3 @@ pub mod apply_cli_overrides; /// pub mod credential_helpers; - -mod _impls { - use git_features::threading::OwnShared; - use std::{ - fmt::{Debug, Formatter}, - ops::{Deref, DerefMut}, - }; - - use crate::config::{CommitAndRollback, Snapshot, SnapshotMut}; - - impl Debug for Snapshot<'_> { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.write_str(&self.repo.config.resolved.to_string()) - } - } - - impl Debug for CommitAndRollback<'_> { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.write_str(&self.repo.config.resolved.to_string()) - } - } - - impl Debug for SnapshotMut<'_> { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.write_str(&self.config.to_string()) - } - } - - impl Drop for SnapshotMut<'_> { - fn drop(&mut self) { - if let Some(repo) = self.repo.take() { - repo.config.resolved = std::mem::take(&mut self.config).into() - }; - } - } - - impl Drop for CommitAndRollback<'_> { - fn drop(&mut self) { - self.repo.config.resolved = OwnShared::clone(&self.prev_config); - } - } - - impl Deref for SnapshotMut<'_> { - type Target = git_config::File<'static>; - - fn deref(&self) -> &Self::Target { - &self.config - } - } - - impl Deref for CommitAndRollback<'_> { - type Target = crate::Repository; - - fn deref(&self) -> &Self::Target { - self.repo - } - } - - impl DerefMut for SnapshotMut<'_> { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.config - } - } -} From 830c45039e9377914bc715002c4e280187498c5c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 27 Sep 2022 19:46:38 +0800 Subject: [PATCH 088/113] read core.excludesFile lazily (and don't cache it yet) (#450) --- git-repository/src/config/cache/access.rs | 17 +++++++++++++++ git-repository/src/config/cache/init.rs | 26 +---------------------- git-repository/src/config/mod.rs | 7 +++--- git-repository/src/worktree/mod.rs | 4 +++- git-repository/tests/repository/mod.rs | 2 +- 5 files changed, 26 insertions(+), 30 deletions(-) diff --git a/git-repository/src/config/cache/access.rs b/git-repository/src/config/cache/access.rs index 67c7f8d6ea6..e751022ee7c 100644 --- a/git-repository/src/config/cache/access.rs +++ b/git-repository/src/config/cache/access.rs @@ -59,6 +59,23 @@ impl Cache { Ok((out[0], out[1])) } + /// The path to the user-level excludes file to ignore certain files in the worktree. + pub(crate) fn excludes_file(&self) -> Result, git_config::path::interpolate::Error> { + let home = self.home_dir(); + let install_dir = crate::path::install_dir().ok(); + let ctx = crate::config::cache::interpolate_context(install_dir.as_deref(), home.as_deref()); + match self + .resolved + .path_filter("core", None, "excludesFile", &mut self.filter_config_section.clone()) + .map(|p| p.interpolate(ctx).map(|p| p.into_owned())) + .transpose() + { + Ok(f) => Ok(f), + Err(_err) if self.lenient_config => Ok(None), + Err(err) => return Err(err), + } + } + /// Return a path by using the `$XDF_CONFIG_HOME` or `$HOME/.config/…` environment variables locations. pub fn xdg_config_path( &self, diff --git a/git-repository/src/config/cache/init.rs b/git-repository/src/config/cache/init.rs index 3fbecadcaaa..72aca29eca1 100644 --- a/git-repository/src/config/cache/init.rs +++ b/git-repository/src/config/cache/init.rs @@ -15,7 +15,7 @@ impl Cache { }: StageOne, git_dir: &std::path::Path, branch_name: Option<&git_ref::FullNameRef>, - mut filter_config_section: fn(&git_config::file::Metadata) -> bool, + filter_config_section: fn(&git_config::file::Metadata) -> bool, git_install_dir: Option<&std::path::Path>, home: Option<&std::path::Path>, repository::permissions::Environment { @@ -111,16 +111,6 @@ impl Cache { globals }; - let excludes_file = match config - .path_filter("core", None, "excludesFile", &mut filter_config_section) - .map(|p| p.interpolate(options.includes.interpolate).map(|p| p.into_owned())) - .transpose() - { - Ok(f) => f, - Err(_err) if lenient_config => None, - Err(err) => return Err(err.into()), - }; - let hex_len = match util::parse_core_abbrev(&config, object_hash) { Ok(v) => v, Err(_err) if lenient_config => None, @@ -143,7 +133,6 @@ impl Cache { ignore_case, hex_len, filter_config_section, - excludes_file, xdg_config_home_env, home_env, lenient_config, @@ -161,19 +150,6 @@ impl Cache { pub fn reread_values_and_clear_caches(&mut self) -> Result<(), Error> { let config = &self.resolved; - let home = self.home_dir(); - let install_dir = crate::path::install_dir().ok(); - let ctx = interpolate_context(install_dir.as_deref(), home.as_deref()); - self.excludes_file = match config - .path_filter("core", None, "excludesFile", &mut self.filter_config_section) - .map(|p| p.interpolate(ctx).map(|p| p.into_owned())) - .transpose() - { - Ok(f) => f, - Err(_err) if self.lenient_config => None, - Err(err) => return Err(err.into()), - }; - self.hex_len = match util::parse_core_abbrev(&config, self.object_hash) { Ok(v) => v, Err(_err) if self.lenient_config => None, diff --git a/git-repository/src/config/mod.rs b/git-repository/src/config/mod.rs index f593e19218e..3eaa8eef60b 100644 --- a/git-repository/src/config/mod.rs +++ b/git-repository/src/config/mod.rs @@ -64,7 +64,10 @@ pub enum Error { PathInterpolation(#[from] git_config::path::interpolate::Error), } -/// Utility type to keep pre-obtained configuration values. +/// Utility type to keep pre-obtained configuration values, only for those required during initial setup +/// and other basic operations that are common enough to warrant a permanent cache. +/// +/// All other values are obtained lazily using OnceCell. #[derive(Clone)] pub(crate) struct Cache { pub resolved: crate::Config, @@ -91,8 +94,6 @@ pub(crate) struct Cache { pub object_kind_hint: Option, /// If true, we are on a case-insensitive file system. pub ignore_case: bool, - /// The path to the user-level excludes file to ignore certain files in the worktree. - pub excludes_file: Option, /// If true, we should default what's possible if something is misconfigured, on case by case basis, to be more resilient. /// Also available in options! Keep in sync! pub lenient_config: bool, diff --git a/git-repository/src/worktree/mod.rs b/git-repository/src/worktree/mod.rs index 1213aa04cca..771036e1fb0 100644 --- a/git-repository/src/worktree/mod.rs +++ b/git-repository/src/worktree/mod.rs @@ -112,6 +112,8 @@ pub mod excludes { Io(#[from] std::io::Error), #[error(transparent)] EnvironmentPermission(#[from] git_sec::permission::Error), + #[error("The value for `core.excludesFile` could not be read from configuration")] + ExcludesFilePathInterpolation(#[from] git_config::path::interpolate::Error), } impl<'repo> crate::Worktree<'repo> { @@ -135,7 +137,7 @@ pub mod excludes { overrides.unwrap_or_default(), git_attributes::MatchGroup::::from_git_dir( repo.git_dir(), - match repo.config.excludes_file.as_ref() { + match repo.config.excludes_file()?.as_ref() { Some(user_path) => Some(user_path.to_owned()), None => repo.config.xdg_config_path("ignore")?, }, diff --git a/git-repository/tests/repository/mod.rs b/git-repository/tests/repository/mod.rs index 6102a04cd2c..481e4fcc3e5 100644 --- a/git-repository/tests/repository/mod.rs +++ b/git-repository/tests/repository/mod.rs @@ -10,7 +10,7 @@ mod worktree; #[test] fn size_in_memory() { - let expected = [768, 808]; + let expected = [728, 784]; let actual_size = std::mem::size_of::(); assert!( expected.contains(&actual_size), From f47a31d7a533c7debc9a44020fa597ff2d48068c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 27 Sep 2022 19:52:28 +0800 Subject: [PATCH 089/113] refactor (#450) --- git-repository/src/config/cache/init.rs | 12 ++---------- git-repository/src/config/cache/util.rs | 8 ++++++++ git-repository/tests/repository/mod.rs | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/git-repository/src/config/cache/init.rs b/git-repository/src/config/cache/init.rs index 72aca29eca1..6720f36da17 100644 --- a/git-repository/src/config/cache/init.rs +++ b/git-repository/src/config/cache/init.rs @@ -111,11 +111,7 @@ impl Cache { globals }; - let hex_len = match util::parse_core_abbrev(&config, object_hash) { - Ok(v) => v, - Err(_err) if lenient_config => None, - Err(err) => return Err(err), - }; + let hex_len = util::check_lenient(util::parse_core_abbrev(&config, object_hash), lenient_config)?; use util::config_bool; let reflog = util::query_refupdates(&config); @@ -150,11 +146,7 @@ impl Cache { pub fn reread_values_and_clear_caches(&mut self) -> Result<(), Error> { let config = &self.resolved; - self.hex_len = match util::parse_core_abbrev(&config, self.object_hash) { - Ok(v) => v, - Err(_err) if self.lenient_config => None, - Err(err) => return Err(err), - }; + self.hex_len = util::check_lenient(util::parse_core_abbrev(&config, self.object_hash), self.lenient_config)?; use util::config_bool; self.ignore_case = config_bool(&config, "core.ignoreCase", false, self.lenient_config)?; diff --git a/git-repository/src/config/cache/util.rs b/git-repository/src/config/cache/util.rs index 9c81b5db74a..1e0c96cbc86 100644 --- a/git-repository/src/config/cache/util.rs +++ b/git-repository/src/config/cache/util.rs @@ -55,6 +55,14 @@ pub(crate) fn query_refupdates(config: &git_config::File<'static>) -> Option(v: Result, E>, lenient: bool) -> Result, E> { + match v { + Ok(v) => Ok(v), + Err(_) if lenient => Ok(None), + Err(err) => Err(err), + } +} + pub(crate) fn parse_core_abbrev( config: &git_config::File<'static>, object_hash: git_hash::Kind, diff --git a/git-repository/tests/repository/mod.rs b/git-repository/tests/repository/mod.rs index 481e4fcc3e5..89c00f76a20 100644 --- a/git-repository/tests/repository/mod.rs +++ b/git-repository/tests/repository/mod.rs @@ -10,7 +10,7 @@ mod worktree; #[test] fn size_in_memory() { - let expected = [728, 784]; + let expected = [728, 744, 784]; let actual_size = std::mem::size_of::(); assert!( expected.contains(&actual_size), From e699291097cec346374a30c325848f787ca9d736 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 27 Sep 2022 20:04:42 +0800 Subject: [PATCH 090/113] change!: `file::Transaction::prepare()` now takes two `git_lock::acquisition::Fail` instances. (#450) This allows to configure the file-ref lock failure mode differently from the packed-refs lock failure mode, which is exactly what `git` does as well defaulting them to 100ms and 1000ms till lock acquisition gives up. --- git-ref/src/store/file/transaction/prepare.rs | 12 +++++++---- .../prepare_and_commit/create_or_update.rs | 21 ++++++++++++++++--- .../transaction/prepare_and_commit/delete.rs | 13 ++++++++++++ git-ref/tests/file/worktree.rs | 16 +++++++++----- 4 files changed, 50 insertions(+), 12 deletions(-) diff --git a/git-ref/src/store/file/transaction/prepare.rs b/git-ref/src/store/file/transaction/prepare.rs index bb402f2a120..2f38dad6ea3 100644 --- a/git-ref/src/store/file/transaction/prepare.rs +++ b/git-ref/src/store/file/transaction/prepare.rs @@ -176,7 +176,8 @@ impl<'s> Transaction<'s> { pub fn prepare( mut self, edits: impl IntoIterator, - lock_fail_mode: git_lock::acquire::Fail, + ref_files_lock_fail_mode: git_lock::acquire::Fail, + packed_refs_lock_fail_mode: git_lock::acquire::Fail, ) -> Result { assert!(self.updates.is_none(), "BUG: Must not call prepare(…) multiple times"); let store = self.store; @@ -267,7 +268,7 @@ impl<'s> Transaction<'s> { let packed_transaction: Option<_> = if maybe_updates_for_packed_refs.unwrap_or(0) > 0 { // We have to create a packed-ref even if it doesn't exist self.store - .packed_transaction(lock_fail_mode) + .packed_transaction(packed_refs_lock_fail_mode) .map_err(|err| match err { file::packed::transaction::Error::BufferOpen(err) => Error::from(err), file::packed::transaction::Error::TransactionLock(err) => { @@ -280,7 +281,10 @@ impl<'s> Transaction<'s> { // no packed-ref file exists anyway self.store .assure_packed_refs_uptodate()? - .map(|p| buffer_into_transaction(p, lock_fail_mode).map_err(Error::PackedTransactionAcquire)) + .map(|p| { + buffer_into_transaction(p, packed_refs_lock_fail_mode) + .map_err(Error::PackedTransactionAcquire) + }) .transpose()? }; if let Some(transaction) = packed_transaction { @@ -302,7 +306,7 @@ impl<'s> Transaction<'s> { let change = &mut updates[cid]; if let Err(err) = Self::lock_ref_and_apply_change( self.store, - lock_fail_mode, + ref_files_lock_fail_mode, self.packed_transaction.as_ref().and_then(|t| t.buffer()), change, ) { diff --git a/git-ref/tests/file/transaction/prepare_and_commit/create_or_update.rs b/git-ref/tests/file/transaction/prepare_and_commit/create_or_update.rs index 666f3517069..352cf53c4cf 100644 --- a/git-ref/tests/file/transaction/prepare_and_commit/create_or_update.rs +++ b/git-ref/tests/file/transaction/prepare_and_commit/create_or_update.rs @@ -44,6 +44,7 @@ fn reference_with_equally_named_empty_or_non_empty_directory_already_in_place_ca deref: false, }), Fail::Immediately, + Fail::Immediately, )? .commit(committer().to_ref()); if *is_empty { @@ -83,6 +84,7 @@ fn reference_with_old_value_must_exist_when_creating_it() -> crate::Result { deref: false, }), Fail::Immediately, + Fail::Immediately, ); match res { @@ -114,6 +116,7 @@ fn reference_with_explicit_value_must_match_the_value_on_update() -> crate::Resu deref: false, }), Fail::Immediately, + Fail::Immediately, ); match res { Err(transaction::prepare::Error::ReferenceOutOfDate { full_name, actual, .. }) => { @@ -142,6 +145,7 @@ fn the_existing_must_match_constraint_allow_non_existing_references_to_be_create deref: false, }), Fail::Immediately, + Fail::Immediately, )? .commit(committer().to_ref())?; @@ -180,6 +184,7 @@ fn the_existing_must_match_constraint_requires_existing_references_to_have_the_g deref: false, }), Fail::Immediately, + Fail::Immediately, ); match res { Err(transaction::prepare::Error::ReferenceOutOfDate { full_name, actual, .. }) => { @@ -208,6 +213,7 @@ fn reference_with_must_not_exist_constraint_cannot_be_created_if_it_exists_alrea deref: false, }), Fail::Immediately, + Fail::Immediately, ); match res { Err(transaction::prepare::Error::MustNotExist { full_name, actual, .. }) => { @@ -246,6 +252,7 @@ fn namespaced_updates_or_deletions_are_transparent_and_not_observable() -> crate }, ], Fail::Immediately, + Fail::Immediately, )? .commit(committer().to_ref())?; @@ -295,6 +302,7 @@ fn reference_with_must_exist_constraint_must_exist_already_with_any_value() -> c deref: false, }), Fail::Immediately, + Fail::Immediately, )? .commit(committer().to_ref())?; @@ -340,6 +348,7 @@ fn reference_with_must_not_exist_constraint_may_exist_already_if_the_new_value_m deref: false, }), Fail::Immediately, + Fail::Immediately, )? .commit(committer().to_ref())?; @@ -387,6 +396,7 @@ fn cancellation_after_preparation_leaves_no_change() -> crate::Result { deref: false, }), Fail::Immediately, + Fail::Immediately, )?; assert_eq!(std::fs::read_dir(dir.path())?.count(), 1, "the lock file was created"); @@ -424,6 +434,7 @@ fn symbolic_head_missing_referent_then_update_referent() -> crate::Result { deref: false, }), Fail::Immediately, + Fail::Immediately, )? .commit(committer().to_ref())?; assert_eq!( @@ -475,6 +486,7 @@ fn symbolic_head_missing_referent_then_update_referent() -> crate::Result { deref: true, }), Fail::Immediately, + Fail::Immediately, )? .commit(committer().to_ref())?; @@ -568,6 +580,7 @@ fn write_reference_to_which_head_points_to_does_not_update_heads_reflog_even_tho deref: false, }), Fail::Immediately, + Fail::Immediately, )? .commit(committer().to_ref())?; @@ -632,7 +645,8 @@ fn packed_refs_are_looked_up_when_checking_existing_values() -> crate::Result { name: "refs/heads/main".try_into()?, deref: false, }), - git_lock::acquire::Fail::Immediately, + Fail::Immediately, + Fail::Immediately, )? .commit(committer().to_ref())?; @@ -687,7 +701,8 @@ fn packed_refs_creation_with_packed_refs_mode_prune_removes_original_loose_refs( name: r.name, deref: false, }), - git_lock::acquire::Fail::Immediately, + Fail::Immediately, + Fail::Immediately, )? .commit(committer().to_ref())?; @@ -743,7 +758,7 @@ fn packed_refs_creation_with_packed_refs_mode_leave_keeps_original_loose_refs() .packed_refs(PackedRefs::DeletionsAndNonSymbolicUpdates(Box::new(|_, _| { Ok(Some(git_object::Kind::Commit)) }))) - .prepare(edits, git_lock::acquire::Fail::Immediately)? + .prepare(edits, Fail::Immediately, Fail::Immediately)? .commit(committer().to_ref())?; assert_eq!( edits.len(), diff --git a/git-ref/tests/file/transaction/prepare_and_commit/delete.rs b/git-ref/tests/file/transaction/prepare_and_commit/delete.rs index 1eecd5fb7f8..026ee137677 100644 --- a/git-ref/tests/file/transaction/prepare_and_commit/delete.rs +++ b/git-ref/tests/file/transaction/prepare_and_commit/delete.rs @@ -28,6 +28,7 @@ fn delete_a_ref_which_is_gone_succeeds() -> crate::Result { deref: false, }), Fail::Immediately, + Fail::Immediately, )? .commit(committer().to_ref())?; assert_eq!(edits.len(), 1); @@ -47,6 +48,7 @@ fn delete_a_ref_which_is_gone_but_must_exist_fails() -> crate::Result { deref: false, }), Fail::Immediately, + Fail::Immediately, ); match res { Ok(_) => unreachable!("must exist, but it doesn't actually exist"), @@ -77,6 +79,7 @@ fn delete_ref_and_reflog_on_symbolic_no_deref() -> crate::Result { deref: false, }), Fail::Immediately, + Fail::Immediately, )? .commit(committer().to_ref())?; @@ -117,6 +120,7 @@ fn delete_ref_with_incorrect_previous_value_fails() -> crate::Result { deref: true, }), Fail::Immediately, + Fail::Immediately, ); match res { @@ -151,6 +155,7 @@ fn delete_reflog_only_of_symbolic_no_deref() -> crate::Result { deref: false, }), Fail::Immediately, + Fail::Immediately, )? .commit(committer().to_ref())?; @@ -185,6 +190,7 @@ fn delete_reflog_only_of_symbolic_with_deref() -> crate::Result { deref: true, }), Fail::Immediately, + Fail::Immediately, )? .commit(committer().to_ref())?; @@ -218,6 +224,7 @@ fn delete_broken_ref_that_must_exist_fails_as_it_is_no_valid_ref() -> crate::Res deref: true, }), Fail::Immediately, + Fail::Immediately, ); match res { Err(err) => { @@ -248,6 +255,7 @@ fn non_existing_can_be_deleted_with_the_may_exist_match_constraint() -> crate::R deref: true, }), Fail::Immediately, + Fail::Immediately, )? .commit(committer().to_ref())?; @@ -284,6 +292,7 @@ fn delete_broken_ref_that_may_not_exist_works_even_in_deref_mode() -> crate::Res deref: true, }), Fail::Immediately, + Fail::Immediately, )? .commit(committer().to_ref())?; @@ -325,6 +334,7 @@ fn store_write_mode_has_no_effect_and_reflogs_are_always_deleted() -> crate::Res deref: false, }), Fail::Immediately, + Fail::Immediately, )? .commit(committer().to_ref())?; assert_eq!(edits.len(), 1); @@ -360,6 +370,7 @@ fn packed_refs_are_consulted_when_determining_previous_value_of_ref_to_be_delete deref: false, }), Fail::Immediately, + Fail::Immediately, )? .commit(committer().to_ref())?; @@ -393,6 +404,7 @@ fn a_loose_ref_with_old_value_check_and_outdated_packed_refs_value_deletes_both_ deref: false, }), Fail::Immediately, + Fail::Immediately, )? .commit(committer().to_ref())?; @@ -432,6 +444,7 @@ fn all_contained_references_deletes_the_packed_ref_file_too() -> crate::Result { } }), Fail::Immediately, + Fail::Immediately, )? .commit(committer().to_ref())?; diff --git a/git-ref/tests/file/worktree.rs b/git-ref/tests/file/worktree.rs index 764fcfd687f..8b39578737a 100644 --- a/git-ref/tests/file/worktree.rs +++ b/git-ref/tests/file/worktree.rs @@ -191,6 +191,7 @@ mod read_only { } mod writable { + use git_lock::acquire::Fail; use std::convert::TryInto; use git_ref::{ @@ -258,7 +259,8 @@ mod writable { deref: false, }, ], - git_lock::acquire::Fail::Immediately, + Fail::Immediately, + Fail::Immediately, )? .commit(committer().to_ref()) .expect("successful commit as even similar resolved names live in different base locations"); @@ -405,7 +407,8 @@ mod writable { deref: false, }, ], - git_lock::acquire::Fail::Immediately, + Fail::Immediately, + Fail::Immediately, ), Err(git_ref::file::transaction::prepare::Error::LockAcquire { .. }) ), "prefixed refs resolve to the same name and will fail to be locked (so we don't check for this when doing dupe checking)"); @@ -424,7 +427,8 @@ mod writable { deref: false, }, ], - git_lock::acquire::Fail::Immediately, + Fail::Immediately, + Fail::Immediately, ), Err(git_ref::file::transaction::prepare::Error::LockAcquire { .. }) )); @@ -467,7 +471,8 @@ mod writable { deref: false, }, ], - git_lock::acquire::Fail::Immediately, + Fail::Immediately, + Fail::Immediately, ), Err(git_ref::file::transaction::prepare::Error::LockAcquire { .. }) ), "prefixed refs resolve to the same name and will fail to be locked (so we don't check for this when doing dupe checking)"); @@ -509,7 +514,8 @@ mod writable { deref: false, }, ], - git_lock::acquire::Fail::Immediately, + Fail::Immediately, + Fail::Immediately, )? .commit(committer().to_ref()) .expect("successful commit as even similar resolved names live in different base locations"); From d40beb3b5744139b56ed68de4caa62a242df2d3a Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 27 Sep 2022 20:06:21 +0800 Subject: [PATCH 091/113] adapt to changes in `git-ref` (#450) --- .../file/init/from_paths/includes/conditional/onbranch.rs | 1 + git-repository/src/config/cache/access.rs | 2 +- git-repository/src/config/cache/init.rs | 6 +++--- git-repository/src/repository/reference.rs | 3 ++- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/git-config/tests/file/init/from_paths/includes/conditional/onbranch.rs b/git-config/tests/file/init/from_paths/includes/conditional/onbranch.rs index f9255841587..38a972ccb7e 100644 --- a/git-config/tests/file/init/from_paths/includes/conditional/onbranch.rs +++ b/git-config/tests/file/init/from_paths/includes/conditional/onbranch.rs @@ -285,6 +285,7 @@ value = branch-override-by-include deref: false, }), git_repository::lock::acquire::Fail::Immediately, + git_repository::lock::acquire::Fail::Immediately, )? .commit(repo.committer_or_default())?; diff --git a/git-repository/src/config/cache/access.rs b/git-repository/src/config/cache/access.rs index e751022ee7c..2f9a630c07d 100644 --- a/git-repository/src/config/cache/access.rs +++ b/git-repository/src/config/cache/access.rs @@ -72,7 +72,7 @@ impl Cache { { Ok(f) => Ok(f), Err(_err) if self.lenient_config => Ok(None), - Err(err) => return Err(err), + Err(err) => Err(err), } } diff --git a/git-repository/src/config/cache/init.rs b/git-repository/src/config/cache/init.rs index 6720f36da17..d576f1c962a 100644 --- a/git-repository/src/config/cache/init.rs +++ b/git-repository/src/config/cache/init.rs @@ -146,11 +146,11 @@ impl Cache { pub fn reread_values_and_clear_caches(&mut self) -> Result<(), Error> { let config = &self.resolved; - self.hex_len = util::check_lenient(util::parse_core_abbrev(&config, self.object_hash), self.lenient_config)?; + self.hex_len = util::check_lenient(util::parse_core_abbrev(config, self.object_hash), self.lenient_config)?; use util::config_bool; - self.ignore_case = config_bool(&config, "core.ignoreCase", false, self.lenient_config)?; - self.object_kind_hint = util::disambiguate_hint(&config); + self.ignore_case = config_bool(config, "core.ignoreCase", false, self.lenient_config)?; + self.object_kind_hint = util::disambiguate_hint(config); self.personas = Default::default(); self.url_rewrite = Default::default(); #[cfg(any(feature = "blocking-network-client", feature = "async-network-client"))] diff --git a/git-repository/src/repository/reference.rs b/git-repository/src/repository/reference.rs index 32e3288cb42..8b8cbbfc931 100644 --- a/git-repository/src/repository/reference.rs +++ b/git-repository/src/repository/reference.rs @@ -132,9 +132,10 @@ impl crate::Repository { &self, edits: impl IntoIterator, ) -> Result, reference::edit::Error> { + let (file_lock_fail, packed_refs_lock_fail) = self.config.lock_timeout()?; self.refs .transaction() - .prepare(edits, self.config.lock_timeout()?.0)? + .prepare(edits, file_lock_fail, packed_refs_lock_fail)? .commit(self.committer_or_default()) .map_err(Into::into) } From fd18320561e05431796aa4044c0a2b0605c9ca9d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 27 Sep 2022 20:20:28 +0800 Subject: [PATCH 092/113] update progress information to include packedRefsTimeout (#450) --- src/plumbing/progress.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plumbing/progress.rs b/src/plumbing/progress.rs index d7ddc9ebf21..5032f23dc9a 100644 --- a/src/plumbing/progress.rs +++ b/src/plumbing/progress.rs @@ -122,7 +122,7 @@ static GIT_CONFIG: &[Record] = &[ }, Record { config: "core.packedRefsTimeout", - usage: Planned {note: Some("needs support in git-ref crate which currently only knows one setting for all locks")}, + usage: InModule {name: "config::cache::access", deviation: None}, }, Record { config: "core.logAllRefUpdates", From cc7564745b45df1848dfb1d7c8f9e1178f0fb64d Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 27 Sep 2022 21:20:50 +0800 Subject: [PATCH 093/113] Add test for commit-and-rollback method, and fix it (#450) --- .../examples/init-repo-and-commit.rs | 2 +- git-repository/src/config/snapshot/_impls.rs | 3 +-- git-repository/src/config/snapshot/access.rs | 27 ++++++++++++++----- .../repository/config/config_snapshot/mod.rs | 16 +++++++++++ 4 files changed, 39 insertions(+), 9 deletions(-) diff --git a/git-repository/examples/init-repo-and-commit.rs b/git-repository/examples/init-repo-and-commit.rs index bcf237d2729..e270b429659 100644 --- a/git-repository/examples/init-repo-and-commit.rs +++ b/git-repository/examples/init-repo-and-commit.rs @@ -24,7 +24,7 @@ fn main() -> anyhow::Result<()> { config.set_raw_value("author", None, "name", "Maria Sanchez")?; config.set_raw_value("author", None, "email", "maria@example.com")?; { - let repo = config.commit_and_rollback(); + let repo = config.commit_and_rollback()?; let initial_commit_id = repo.commit("HEAD", "initial commit", empty_tree_id, git::commit::NO_PARENT_IDS)?; println!("initial commit id with empty tree: {:?}", initial_commit_id); diff --git a/git-repository/src/config/snapshot/_impls.rs b/git-repository/src/config/snapshot/_impls.rs index b449addf63d..6406b233c30 100644 --- a/git-repository/src/config/snapshot/_impls.rs +++ b/git-repository/src/config/snapshot/_impls.rs @@ -27,8 +27,7 @@ impl Debug for SnapshotMut<'_> { impl Drop for SnapshotMut<'_> { fn drop(&mut self) { if let Some(repo) = self.repo.take() { - repo.config.resolved = std::mem::take(&mut self.config).into(); - repo.config.reread_values_and_clear_caches().ok(); + self.commit_inner(repo).ok(); }; } } diff --git a/git-repository/src/config/snapshot/access.rs b/git-repository/src/config/snapshot/access.rs index 7ccd5b25f4c..fe8c9754259 100644 --- a/git-repository/src/config/snapshot/access.rs +++ b/git-repository/src/config/snapshot/access.rs @@ -97,16 +97,31 @@ impl<'repo> Snapshot<'repo> { impl<'repo> SnapshotMut<'repo> { /// Apply all changes made to this instance. /// - /// Note that this would also happen once this instance is dropped, but using this method may be more intuitive. - pub fn commit(self) {} + /// Note that this would also happen once this instance is dropped, but using this method may be more intuitive and won't squelch errors + /// in case the new configuration is partially invalid. + pub fn commit(mut self) -> Result<&'repo mut crate::Repository, crate::config::Error> { + let repo = self.repo.take().expect("always present here"); + self.commit_inner(repo) + } + + pub(crate) fn commit_inner( + &mut self, + repo: &'repo mut crate::Repository, + ) -> Result<&'repo mut crate::Repository, crate::config::Error> { + repo.config.resolved = std::mem::take(&mut self.config).into(); + repo.config.reread_values_and_clear_caches()?; + Ok(repo) + } /// Create a structure the temporarily commits the changes, but rolls them back when dropped. - pub fn commit_and_rollback(mut self) -> CommitAndRollback<'repo> { - let new_config = std::mem::take(&mut self.config); + pub fn commit_and_rollback(mut self) -> Result, crate::config::Error> { let repo = self.repo.take().expect("this only runs once on consumption"); - repo.config.resolved = new_config.into(); let prev_config = OwnShared::clone(&repo.config.resolved); - CommitAndRollback { repo, prev_config } + + Ok(CommitAndRollback { + repo: self.commit_inner(repo)?, + prev_config, + }) } /// Don't apply any of the changes after consuming this instance, effectively forgetting them, returning the changed configuration. diff --git a/git-repository/tests/repository/config/config_snapshot/mod.rs b/git-repository/tests/repository/config/config_snapshot/mod.rs index 8eb326a843b..ab3ccfbcccd 100644 --- a/git-repository/tests/repository/config/config_snapshot/mod.rs +++ b/git-repository/tests/repository/config/config_snapshot/mod.rs @@ -1,5 +1,21 @@ use crate::named_repo; +#[test] +fn commit_and_rollback() -> crate::Result { + let mut repo: git_repository::Repository = named_repo("make_basic_repo.sh")?; + assert_eq!(repo.head_id()?.shorten()?.to_string(), "3189cd3"); + + { + let mut repo = repo.config_snapshot_mut(); + repo.set_raw_value("core", None, "abbrev", "4")?; + let repo = repo.commit_and_rollback()?; + assert_eq!(repo.head_id()?.shorten()?.to_string(), "3189"); + } + + assert_eq!(repo.head_id()?.shorten()?.to_string(), "3189cd3"); + Ok(()) +} + #[test] fn values_are_set_in_memory_only() { let mut repo = named_repo("make_config_repo.sh").unwrap(); From b5149661d9160540359b19c204388c87c778727f Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 27 Sep 2022 21:27:16 +0800 Subject: [PATCH 094/113] more robust assignment of re-evaluated cached values (#450) --- git-repository/src/config/cache/init.rs | 20 +++++++++++++------- git-repository/src/config/snapshot/_impls.rs | 6 ++++-- git-repository/src/config/snapshot/access.rs | 4 ++-- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/git-repository/src/config/cache/init.rs b/git-repository/src/config/cache/init.rs index d576f1c962a..01442d25d09 100644 --- a/git-repository/src/config/cache/init.rs +++ b/git-repository/src/config/cache/init.rs @@ -140,17 +140,18 @@ impl Cache { }) } - /// Call this after the `resolved` configuration changed in a way that may affect the caches provided here. + /// Call this with new `config` to update values and clear caches. Note that none of the values will be applied if a single + /// one is invalid. + /// However, those that are lazily read won't be re-evaluated right away and might thus pass now but fail later. /// /// Note that we unconditionally re-read all values. - pub fn reread_values_and_clear_caches(&mut self) -> Result<(), Error> { - let config = &self.resolved; - - self.hex_len = util::check_lenient(util::parse_core_abbrev(config, self.object_hash), self.lenient_config)?; + pub fn reread_values_and_clear_caches(&mut self, config: crate::Config) -> Result<(), Error> { + let hex_len = util::check_lenient(util::parse_core_abbrev(&config, self.object_hash), self.lenient_config)?; use util::config_bool; - self.ignore_case = config_bool(config, "core.ignoreCase", false, self.lenient_config)?; - self.object_kind_hint = util::disambiguate_hint(config); + let ignore_case = config_bool(&config, "core.ignoreCase", false, self.lenient_config)?; + let object_kind_hint = util::disambiguate_hint(&config); + self.personas = Default::default(); self.url_rewrite = Default::default(); #[cfg(any(feature = "blocking-network-client", feature = "async-network-client"))] @@ -158,6 +159,11 @@ impl Cache { self.url_scheme = Default::default(); } + self.resolved = config; + self.hex_len = hex_len; + self.ignore_case = ignore_case; + self.object_kind_hint = object_kind_hint; + Ok(()) } } diff --git a/git-repository/src/config/snapshot/_impls.rs b/git-repository/src/config/snapshot/_impls.rs index 6406b233c30..de470b8520d 100644 --- a/git-repository/src/config/snapshot/_impls.rs +++ b/git-repository/src/config/snapshot/_impls.rs @@ -34,8 +34,10 @@ impl Drop for SnapshotMut<'_> { impl Drop for CommitAndRollback<'_> { fn drop(&mut self) { - self.repo.config.resolved = OwnShared::clone(&self.prev_config); - self.repo.config.reread_values_and_clear_caches().ok(); + self.repo + .config + .reread_values_and_clear_caches(OwnShared::clone(&self.prev_config)) + .ok(); } } diff --git a/git-repository/src/config/snapshot/access.rs b/git-repository/src/config/snapshot/access.rs index fe8c9754259..16e149523b7 100644 --- a/git-repository/src/config/snapshot/access.rs +++ b/git-repository/src/config/snapshot/access.rs @@ -108,8 +108,8 @@ impl<'repo> SnapshotMut<'repo> { &mut self, repo: &'repo mut crate::Repository, ) -> Result<&'repo mut crate::Repository, crate::config::Error> { - repo.config.resolved = std::mem::take(&mut self.config).into(); - repo.config.reread_values_and_clear_caches()?; + repo.config + .reread_values_and_clear_caches(std::mem::take(&mut self.config).into())?; Ok(repo) } From dde9e6345f53e36d6a9528d4132b16a6659999dd Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 27 Sep 2022 21:42:26 +0800 Subject: [PATCH 095/113] more tests for SnapshotMut and now it's working properly (#450) --- .../examples/init-repo-and-commit.rs | 2 +- git-repository/src/config/mod.rs | 4 +-- git-repository/src/config/snapshot/_impls.rs | 20 ++++++------ git-repository/src/config/snapshot/access.rs | 27 +++++++++++++--- .../repository/config/config_snapshot/mod.rs | 32 +++++++++++++++++-- 5 files changed, 65 insertions(+), 20 deletions(-) diff --git a/git-repository/examples/init-repo-and-commit.rs b/git-repository/examples/init-repo-and-commit.rs index e270b429659..1b0817b57dc 100644 --- a/git-repository/examples/init-repo-and-commit.rs +++ b/git-repository/examples/init-repo-and-commit.rs @@ -24,7 +24,7 @@ fn main() -> anyhow::Result<()> { config.set_raw_value("author", None, "name", "Maria Sanchez")?; config.set_raw_value("author", None, "email", "maria@example.com")?; { - let repo = config.commit_and_rollback()?; + let repo = config.commit_auto_rollback()?; let initial_commit_id = repo.commit("HEAD", "initial commit", empty_tree_id, git::commit::NO_PARENT_IDS)?; println!("initial commit id with empty tree: {:?}", initial_commit_id); diff --git a/git-repository/src/config/mod.rs b/git-repository/src/config/mod.rs index 3eaa8eef60b..f08aba3e9d3 100644 --- a/git-repository/src/config/mod.rs +++ b/git-repository/src/config/mod.rs @@ -29,8 +29,8 @@ pub struct SnapshotMut<'repo> { } /// A utility structure created by [`SnapshotMut::commit_and_rollback()`] that restores the previous configuration on drop. -pub struct CommitAndRollback<'repo> { - pub(crate) repo: &'repo mut Repository, +pub struct CommitAutoRollback<'repo> { + pub(crate) repo: Option<&'repo mut Repository>, pub(crate) prev_config: crate::Config, } diff --git a/git-repository/src/config/snapshot/_impls.rs b/git-repository/src/config/snapshot/_impls.rs index de470b8520d..91332ad1652 100644 --- a/git-repository/src/config/snapshot/_impls.rs +++ b/git-repository/src/config/snapshot/_impls.rs @@ -1,10 +1,9 @@ -use git_features::threading::OwnShared; use std::{ fmt::{Debug, Formatter}, ops::{Deref, DerefMut}, }; -use crate::config::{CommitAndRollback, Snapshot, SnapshotMut}; +use crate::config::{CommitAutoRollback, Snapshot, SnapshotMut}; impl Debug for Snapshot<'_> { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { @@ -12,9 +11,9 @@ impl Debug for Snapshot<'_> { } } -impl Debug for CommitAndRollback<'_> { +impl Debug for CommitAutoRollback<'_> { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.write_str(&self.repo.config.resolved.to_string()) + f.write_str(&self.repo.as_ref().expect("still present").config.resolved.to_string()) } } @@ -32,12 +31,11 @@ impl Drop for SnapshotMut<'_> { } } -impl Drop for CommitAndRollback<'_> { +impl Drop for CommitAutoRollback<'_> { fn drop(&mut self) { - self.repo - .config - .reread_values_and_clear_caches(OwnShared::clone(&self.prev_config)) - .ok(); + if let Some(repo) = self.repo.take() { + self.rollback_inner(repo).ok(); + } } } @@ -49,11 +47,11 @@ impl Deref for SnapshotMut<'_> { } } -impl Deref for CommitAndRollback<'_> { +impl Deref for CommitAutoRollback<'_> { type Target = crate::Repository; fn deref(&self) -> &Self::Target { - self.repo + self.repo.as_ref().expect("always present") } } diff --git a/git-repository/src/config/snapshot/access.rs b/git-repository/src/config/snapshot/access.rs index 16e149523b7..106f369ca9d 100644 --- a/git-repository/src/config/snapshot/access.rs +++ b/git-repository/src/config/snapshot/access.rs @@ -1,7 +1,7 @@ use git_features::threading::OwnShared; use std::borrow::Cow; -use crate::config::{CommitAndRollback, SnapshotMut}; +use crate::config::{CommitAutoRollback, SnapshotMut}; use crate::{ bstr::BStr, config::{cache::interpolate_context, Snapshot}, @@ -114,18 +114,37 @@ impl<'repo> SnapshotMut<'repo> { } /// Create a structure the temporarily commits the changes, but rolls them back when dropped. - pub fn commit_and_rollback(mut self) -> Result, crate::config::Error> { + pub fn commit_auto_rollback(mut self) -> Result, crate::config::Error> { let repo = self.repo.take().expect("this only runs once on consumption"); let prev_config = OwnShared::clone(&repo.config.resolved); - Ok(CommitAndRollback { - repo: self.commit_inner(repo)?, + Ok(CommitAutoRollback { + repo: self.commit_inner(repo)?.into(), prev_config, }) } /// Don't apply any of the changes after consuming this instance, effectively forgetting them, returning the changed configuration. pub fn forget(mut self) -> git_config::File<'static> { + self.repo.take(); std::mem::take(&mut self.config) } } + +/// Utilities +impl<'repo> CommitAutoRollback<'repo> { + /// Rollback the changes previously applied and all values before the change. + pub fn rollback(mut self) -> Result<&'repo mut crate::Repository, crate::config::Error> { + let repo = self.repo.take().expect("still present, consumed only once"); + self.rollback_inner(repo) + } + + pub(crate) fn rollback_inner( + &mut self, + repo: &'repo mut crate::Repository, + ) -> Result<&'repo mut crate::Repository, crate::config::Error> { + repo.config + .reread_values_and_clear_caches(OwnShared::clone(&self.prev_config))?; + Ok(repo) + } +} diff --git a/git-repository/tests/repository/config/config_snapshot/mod.rs b/git-repository/tests/repository/config/config_snapshot/mod.rs index ab3ccfbcccd..cf46f53f5a4 100644 --- a/git-repository/tests/repository/config/config_snapshot/mod.rs +++ b/git-repository/tests/repository/config/config_snapshot/mod.rs @@ -1,18 +1,46 @@ use crate::named_repo; #[test] -fn commit_and_rollback() -> crate::Result { +fn commit_auto_rollback() -> crate::Result { let mut repo: git_repository::Repository = named_repo("make_basic_repo.sh")?; assert_eq!(repo.head_id()?.shorten()?.to_string(), "3189cd3"); { let mut repo = repo.config_snapshot_mut(); repo.set_raw_value("core", None, "abbrev", "4")?; - let repo = repo.commit_and_rollback()?; + let repo = repo.commit_auto_rollback()?; assert_eq!(repo.head_id()?.shorten()?.to_string(), "3189"); } assert_eq!(repo.head_id()?.shorten()?.to_string(), "3189cd3"); + + let repo = { + let mut repo = repo.config_snapshot_mut(); + repo.set_raw_value("core", None, "abbrev", "4")?; + let repo = repo.commit_auto_rollback()?; + assert_eq!(repo.head_id()?.shorten()?.to_string(), "3189"); + repo.rollback()? + }; + assert_eq!(repo.head_id()?.shorten()?.to_string(), "3189cd3"); + + Ok(()) +} + +#[test] +fn snapshot_mut_commit_and_forget() -> crate::Result { + let mut repo: git_repository::Repository = named_repo("make_basic_repo.sh")?; + let repo = { + let mut repo = repo.config_snapshot_mut(); + repo.set_raw_value("core", None, "abbrev", "4")?; + repo.commit()? + }; + assert_eq!(repo.config_snapshot().integer("core.abbrev").expect("set"), 4); + { + let mut repo = repo.config_snapshot_mut(); + repo.set_raw_value("core", None, "abbrev", "8")?; + repo.forget(); + } + assert_eq!(repo.config_snapshot().integer("core.abbrev"), Some(4)); Ok(()) } From 005469cdaef4defb35ae65c23962c9f7da98c12f Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 27 Sep 2022 21:49:35 +0800 Subject: [PATCH 096/113] isolate test properly (#450) Previously it picked up the users configuration as it was loading the standard git configuration, instead of the restricted one. --- git-repository/tests/repository/object.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/git-repository/tests/repository/object.rs b/git-repository/tests/repository/object.rs index 113177f44b2..8b9f7a31cc7 100644 --- a/git-repository/tests/repository/object.rs +++ b/git-repository/tests/repository/object.rs @@ -147,7 +147,7 @@ mod tag { } mod commit { - use crate::freeze_time; + use crate::{freeze_time, restricted}; use git_repository as git; use git_testtools::hex_to_id; @@ -170,12 +170,12 @@ mod commit { fn single_line_initial_commit_empty_tree_ref_nonexisting() -> crate::Result { freeze_time(); let tmp = tempfile::tempdir()?; - let repo = git::init(&tmp)?; + let repo = git::open_opts(git::init(&tmp)?.path(), restricted())?; let empty_tree_id = repo.write_object(&git::objs::Tree::empty())?; let commit_id = repo.commit("HEAD", "initial", empty_tree_id, git::commit::NO_PARENT_IDS)?; assert_eq!( commit_id, - hex_to_id("bbbd91e8b0f81f0693291d4a47c3f2c724ac44ee"), + hex_to_id("3a774843723a713a8d361b4d4d98ad4092ef05bd"), "the commit id is stable" ); From 6981b71a23444827dff4dfe6cc5f1c04beceddc1 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 27 Sep 2022 21:55:48 +0800 Subject: [PATCH 097/113] more robust tests that depend on time. (#450) Unfortunately all tests are now liable to picking up GIT_ environment variables which will affect those that create commits. Maybe better control them right away or find another way to set the time. --- git-repository/tests/repository/object.rs | 6 ++++-- git-repository/tests/util/mod.rs | 12 +++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/git-repository/tests/repository/object.rs b/git-repository/tests/repository/object.rs index 8b9f7a31cc7..6c0b793f73a 100644 --- a/git-repository/tests/repository/object.rs +++ b/git-repository/tests/repository/object.rs @@ -167,8 +167,9 @@ mod commit { } #[test] + #[serial_test::serial] fn single_line_initial_commit_empty_tree_ref_nonexisting() -> crate::Result { - freeze_time(); + let _env = freeze_time(); let tmp = tempfile::tempdir()?; let repo = git::open_opts(git::init(&tmp)?.path(), restricted())?; let empty_tree_id = repo.write_object(&git::objs::Tree::empty())?; @@ -194,8 +195,9 @@ mod commit { } #[test] + #[serial_test::serial] fn multi_line_commit_message_uses_first_line_in_ref_log_ref_nonexisting() -> crate::Result { - freeze_time(); + let _env = freeze_time(); let (repo, _keep) = crate::basic_rw_repo()?; let parent = repo.find_reference("HEAD")?.peel_to_id_in_place()?; let empty_tree_id = parent.object()?.to_commit_ref_iter().tree_id().expect("tree to be set"); diff --git a/git-repository/tests/util/mod.rs b/git-repository/tests/util/mod.rs index 8710c7cd91d..1ea8f02d84d 100644 --- a/git-repository/tests/util/mod.rs +++ b/git-repository/tests/util/mod.rs @@ -1,14 +1,12 @@ use git_repository::{open, Repository, ThreadSafeRepository}; -use once_cell::sync::Lazy; pub type Result = std::result::Result>; -pub fn freeze_time() { - static FROZEN: Lazy<()> = Lazy::new(|| { - std::env::set_var("GIT_AUTHOR_DATE", "1979-02-26 18:30:00"); - std::env::set_var("GIT_COMMITTER_DATE", "1979-02-26 18:30:00"); - }); - *FROZEN +pub fn freeze_time() -> git_testtools::Env<'static> { + let frozen_time = "1979-02-26 18:30:00"; + git_testtools::Env::new() + .set("GIT_AUTHOR_DATE", frozen_time) + .set("GIT_COMMITTER_DATE", frozen_time) } pub fn repo(name: &str) -> Result { let repo_path = git_testtools::scripted_fixture_repo_read_only(name)?; From 09da4c5eeff5c6657beb9c53c168f90e74d6f758 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 27 Sep 2022 22:07:49 +0800 Subject: [PATCH 098/113] feat: add `Env::unset()` for convenience (#450) --- tests/tools/src/lib.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/tools/src/lib.rs b/tests/tools/src/lib.rs index 5bdc474bbf9..853f02752ae 100644 --- a/tests/tools/src/lib.rs +++ b/tests/tools/src/lib.rs @@ -535,6 +535,14 @@ impl<'a> Env<'a> { self.altered_vars.push((var, prev)); self } + + /// Set `var` to `value`. + pub fn unset(mut self, var: &'a str) -> Self { + let prev = std::env::var_os(var); + std::env::remove_var(var); + self.altered_vars.push((var, prev)); + self + } } impl<'a> Drop for Env<'a> { From 25d61067640016b21cdf1eb90998be512b805e8b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 27 Sep 2022 22:08:32 +0800 Subject: [PATCH 099/113] unset other variables that we know may affect some functions that need stability. (#450) Note that this is only required as we freeze time using environment variables, so `GIT_` is now allowed when opening repos. There would be another way, which is controlling exactly what restricted means. --- git-repository/tests/util/mod.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/git-repository/tests/util/mod.rs b/git-repository/tests/util/mod.rs index 1ea8f02d84d..d36ad143720 100644 --- a/git-repository/tests/util/mod.rs +++ b/git-repository/tests/util/mod.rs @@ -5,7 +5,11 @@ pub type Result = std::result::Result>; pub fn freeze_time() -> git_testtools::Env<'static> { let frozen_time = "1979-02-26 18:30:00"; git_testtools::Env::new() + .unset("GIT_AUTHOR_NAME") + .unset("GIT_AUTHOR_EMAIL") .set("GIT_AUTHOR_DATE", frozen_time) + .unset("GIT_COMMITTER_NAME") + .unset("GIT_COMMITTER_EMAIL") .set("GIT_COMMITTER_DATE", frozen_time) } pub fn repo(name: &str) -> Result { From 9704c2f075af05bf258854b378ed91b8a5d71e93 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 27 Sep 2022 22:14:12 +0800 Subject: [PATCH 100/113] further harden tests to only allow environment variables for those who need it. (#450) This means, other tests are once again isolated by default --- git-repository/tests/repository/object.rs | 6 +++--- git-repository/tests/util/mod.rs | 12 ++++++++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/git-repository/tests/repository/object.rs b/git-repository/tests/repository/object.rs index 6c0b793f73a..fffc17c867d 100644 --- a/git-repository/tests/repository/object.rs +++ b/git-repository/tests/repository/object.rs @@ -147,7 +147,7 @@ mod tag { } mod commit { - use crate::{freeze_time, restricted}; + use crate::{freeze_time, restricted_and_git}; use git_repository as git; use git_testtools::hex_to_id; @@ -171,7 +171,7 @@ mod commit { fn single_line_initial_commit_empty_tree_ref_nonexisting() -> crate::Result { let _env = freeze_time(); let tmp = tempfile::tempdir()?; - let repo = git::open_opts(git::init(&tmp)?.path(), restricted())?; + let repo = git::open_opts(git::init(&tmp)?.path(), restricted_and_git())?; let empty_tree_id = repo.write_object(&git::objs::Tree::empty())?; let commit_id = repo.commit("HEAD", "initial", empty_tree_id, git::commit::NO_PARENT_IDS)?; assert_eq!( @@ -198,7 +198,7 @@ mod commit { #[serial_test::serial] fn multi_line_commit_message_uses_first_line_in_ref_log_ref_nonexisting() -> crate::Result { let _env = freeze_time(); - let (repo, _keep) = crate::basic_rw_repo()?; + let (repo, _keep) = crate::repo_rw_opts("make_basic_repo.sh", restricted_and_git())?; let parent = repo.find_reference("HEAD")?.peel_to_id_in_place()?; let empty_tree_id = parent.object()?.to_commit_ref_iter().tree_id().expect("tree to be set"); assert_eq!( diff --git a/git-repository/tests/util/mod.rs b/git-repository/tests/util/mod.rs index d36ad143720..e51af863ddb 100644 --- a/git-repository/tests/util/mod.rs +++ b/git-repository/tests/util/mod.rs @@ -23,20 +23,28 @@ pub fn named_repo(name: &str) -> Result { } pub fn restricted() -> open::Options { + open::Options::isolated() +} + +pub fn restricted_and_git() -> open::Options { let mut opts = open::Options::isolated(); opts.permissions.env.git_prefix = git_sec::Permission::Allow; opts } pub fn repo_rw(name: &str) -> Result<(Repository, tempfile::TempDir)> { + repo_rw_opts(name, restricted()) +} + +pub fn repo_rw_opts(name: &str, opts: git_repository::open::Options) -> Result<(Repository, tempfile::TempDir)> { let repo_path = git_testtools::scripted_fixture_repo_writable(name)?; Ok(( ThreadSafeRepository::discover_opts( repo_path.path(), Default::default(), git_sec::trust::Mapping { - full: restricted(), - reduced: restricted(), + full: opts.clone(), + reduced: opts, }, )? .to_thread_local(), From 2a675312b7a4d28cc84e09d54e1f929b0e56f75f Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Tue, 27 Sep 2022 22:15:56 +0800 Subject: [PATCH 101/113] fix docs (#450) --- git-repository/src/config/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git-repository/src/config/mod.rs b/git-repository/src/config/mod.rs index f08aba3e9d3..bdfe59a62b0 100644 --- a/git-repository/src/config/mod.rs +++ b/git-repository/src/config/mod.rs @@ -28,7 +28,7 @@ pub struct SnapshotMut<'repo> { pub(crate) config: git_config::File<'static>, } -/// A utility structure created by [`SnapshotMut::commit_and_rollback()`] that restores the previous configuration on drop. +/// A utility structure created by [`SnapshotMut::commit_auto_rollback()`] that restores the previous configuration on drop. pub struct CommitAutoRollback<'repo> { pub(crate) repo: Option<&'repo mut Repository>, pub(crate) prev_config: crate::Config, From 870b6806f0199b13916f05f46a22fdd3e2a76513 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 28 Sep 2022 20:33:25 +0800 Subject: [PATCH 102/113] add failing test to validate worktree-checkout check (#450) --- .../connection/fetch/update_refs/mod.rs | 2 +- .../connection/fetch/update_refs/tests.rs | 68 +++++++++++++++---- .../connection/fetch/update_refs/update.rs | 12 +++- .../tests/fixtures/make_fetch_repos.sh | 12 ++++ 4 files changed, 75 insertions(+), 19 deletions(-) diff --git a/git-repository/src/remote/connection/fetch/update_refs/mod.rs b/git-repository/src/remote/connection/fetch/update_refs/mod.rs index 44488e93e6d..34bfe75669f 100644 --- a/git-repository/src/remote/connection/fetch/update_refs/mod.rs +++ b/git-repository/src/remote/connection/fetch/update_refs/mod.rs @@ -7,7 +7,7 @@ use std::convert::TryInto; pub mod update; /// Information about the update of a single reference, corresponding the respective entry in [`RefMap::mappings`][crate::remote::fetch::RefMap::mappings]. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct Update { /// The way the update was performed. pub mode: update::Mode, diff --git a/git-repository/src/remote/connection/fetch/update_refs/tests.rs b/git-repository/src/remote/connection/fetch/update_refs/tests.rs index ced23a2181d..d4d60bceddd 100644 --- a/git-repository/src/remote/connection/fetch/update_refs/tests.rs +++ b/git-repository/src/remote/connection/fetch/update_refs/tests.rs @@ -1,22 +1,22 @@ mod update { - use crate as git; - fn repo(name: &str) -> git::Repository { - let dir = git_testtools::scripted_fixture_repo_read_only_with_args( - "make_fetch_repos.sh", - [git::path::realpath( - git_testtools::scripted_fixture_repo_read_only("make_remote_repos.sh") - .unwrap() - .join("base"), - ) - .unwrap() - .to_string_lossy()], + fn base_repo_path() -> String { + git::path::realpath( + git_testtools::scripted_fixture_repo_read_only("make_remote_repos.sh") + .unwrap() + .join("base"), ) - .unwrap(); - git::open_opts(dir.join(name), git::open::Options::isolated()).unwrap() + .unwrap() + .to_string_lossy() + .into_owned() } + fn repo(name: &str) -> git::Repository { + let dir = git_testtools::scripted_fixture_repo_read_only_with_args("make_fetch_repos.sh", [base_repo_path()]) + .unwrap(); + git::open_opts(dir.join(name), git::open::Options::isolated()).unwrap() + } use crate::remote::fetch; use git_ref::TargetRef; @@ -58,7 +58,9 @@ mod update { ), ( "+refs/remotes/origin/g:refs/heads/main", - fetch::refs::update::Mode::RejectedCheckedOut, + fetch::refs::update::Mode::RejectedCurrentlyCheckedOut { + worktree_dir: repo.work_dir().expect("present").to_owned(), + }, false, "checked out branches cannot be written, as it requires a merge of sorts which isn't done here", ), @@ -75,7 +77,7 @@ mod update { assert_eq!( out.updates, vec![fetch::refs::Update { - mode: expected_mode, + mode: expected_mode.clone(), edit_index: has_edit_index.then(|| 0), spec_index: 0 }], @@ -85,6 +87,42 @@ mod update { } } + #[test] + #[ignore] + fn checked_out_branches_in_worktrees_are_rejected_with_additional_infromation() { + let root = git_testtools::scripted_fixture_repo_read_only_with_args("make_fetch_repos.sh", [base_repo_path()]) + .unwrap(); + let repo = root.join("worktree-root"); + let repo = git::open_opts(repo, git::open::Options::isolated()).unwrap(); + for (branch, path_from_root) in [ + ("main", "worktree-root"), + ("wt-a-nested", "prec/wt-a"), + ("wt-a", "wt-a"), + ("nested-wt-b", "wt-a/nested-wt-b"), + ("wt-c-locked", "wt-c-locked"), + ("wt-deleted", "wt-deleted"), + ] { + let spec = format!("refs/heads/main:refs/heads/{}", branch); + let (mappings, specs) = mapping_from_spec(&spec, &repo); + let out = fetch::refs::update(&repo, &mappings, &specs, fetch::DryRun::Yes).unwrap(); + + assert_eq!( + out.updates, + vec![fetch::refs::Update { + mode: fetch::refs::update::Mode::RejectedCurrentlyCheckedOut { + worktree_dir: root.join(path_from_root), + }, + edit_index: None, + spec_index: 0, + }], + "{}: checked-out checks are done before checking if a change would actually be required (here it isn't)", spec + ); + + // TODO: add + assert_eq!(out.edits.len(), 0); + } + } + #[test] fn symbolic_refs_are_never_written() { let repo = repo("two-origins"); diff --git a/git-repository/src/remote/connection/fetch/update_refs/update.rs b/git-repository/src/remote/connection/fetch/update_refs/update.rs index 39952af57dd..eae2ec85634 100644 --- a/git-repository/src/remote/connection/fetch/update_refs/update.rs +++ b/git-repository/src/remote/connection/fetch/update_refs/update.rs @@ -1,4 +1,5 @@ use crate::remote::fetch; +use std::path::PathBuf; mod error { /// The error returned when updating references. @@ -28,7 +29,7 @@ pub struct Outcome { } /// Describe the way a ref was updated -#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub enum Mode { /// The old ref's commit was an ancestor of the new one, allowing for a fast-forward without a merge. FastForward, @@ -40,8 +41,13 @@ pub enum Mode { RejectedNonFastForward, /// The update of a local symbolic reference was rejected. RejectedSymbolic, - /// The update was rejected as the destination branch is currently checked out - RejectedCheckedOut, + /// The update was rejected because the branch is checked out in the given worktree_dir. + /// + /// Note that the check applies to any known worktree, whether it's present on disk or not. + RejectedCurrentlyCheckedOut { + /// The path to the worktree directory where the branch is checked out. + worktree_dir: PathBuf, + }, /// No change was attempted as the remote ref didn't change compared to the current ref, or because no remote ref was specified /// in the ref-spec. NoChangeNeeded, diff --git a/git-repository/tests/fixtures/make_fetch_repos.sh b/git-repository/tests/fixtures/make_fetch_repos.sh index 934f9cc9681..d26e3a46660 100644 --- a/git-repository/tests/fixtures/make_fetch_repos.sh +++ b/git-repository/tests/fixtures/make_fetch_repos.sh @@ -16,3 +16,15 @@ git clone --shared base two-origins git branch "not-currently-checked-out" git symbolic-ref refs/heads/symbolic refs/heads/main ) + +git clone --shared base worktree-root +( + cd worktree-root + + git worktree add ../wt-a + git worktree add ../prev/wt-a-nested + git worktree add ../wt-b + git worktree add ../wt-a/nested-wt-b + git worktree add --lock ../wt-c-locked + git worktree add ../wt-deleted && rm -Rf ../wt-deleted +) From 5a6c1025012d63e019e8b554078a33d23190bf18 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Wed, 28 Sep 2022 21:02:44 +0800 Subject: [PATCH 103/113] reject updating checked-out branches (#450) This is tested for worktrees as well, including deleted ones, and validated by hand the git produces the same results. --- .../connection/fetch/update_refs/mod.rs | 58 ++++++++++++++----- .../connection/fetch/update_refs/tests.rs | 19 +++--- .../connection/fetch/update_refs/update.rs | 4 ++ git-repository/src/repository/worktree.rs | 12 +--- 4 files changed, 59 insertions(+), 34 deletions(-) diff --git a/git-repository/src/remote/connection/fetch/update_refs/mod.rs b/git-repository/src/remote/connection/fetch/update_refs/mod.rs index 34bfe75669f..5c621e654f4 100644 --- a/git-repository/src/remote/connection/fetch/update_refs/mod.rs +++ b/git-repository/src/remote/connection/fetch/update_refs/mod.rs @@ -1,7 +1,10 @@ use crate::remote::fetch; +use crate::Repository; use git_ref::transaction::{Change, LogChange, PreviousValue, RefEdit, RefLog}; use git_ref::{Target, TargetRef}; +use std::collections::BTreeMap; use std::convert::TryInto; +use std::path::PathBuf; /// pub mod update; @@ -23,7 +26,7 @@ pub struct Update { /// /// It can be used to produce typical information that one is used to from `git fetch`. pub(crate) fn update( - repo: &crate::Repository, + repo: &Repository, mappings: &[fetch::Mapping], refspecs: &[git_refspec::RefSpec], dry_run: fetch::DryRun, @@ -38,29 +41,42 @@ pub(crate) fn update( } in mappings { let remote_id = remote.as_id(); + let checked_out_branches = worktree_branches(repo)?; let (mode, edit_index) = match local { Some(name) => { let (mode, reflog_message, name) = match repo.try_find_reference(name)? { - Some(existing) => match existing.target() { - TargetRef::Symbolic(_) => { + Some(existing) => { + if let Some(wt_dir) = checked_out_branches.get(existing.name()) { updates.push(Update { - mode: update::Mode::RejectedSymbolic, + mode: update::Mode::RejectedCurrentlyCheckedOut { + worktree_dir: wt_dir.to_owned(), + }, spec_index: *spec_index, edit_index: None, }); continue; } - TargetRef::Peeled(local_id) => { - let (mode, reflog_message) = if local_id == remote_id { - (update::Mode::NoChangeNeeded, "TBD no change") - } else if refspecs[*spec_index].allow_non_fast_forward() { - (update::Mode::Forced, "TBD force") - } else { - todo!("check for fast-forward (is local an ancestor of remote?)") - }; - (mode, reflog_message, existing.name().to_owned()) + match existing.target() { + TargetRef::Symbolic(_) => { + updates.push(Update { + mode: update::Mode::RejectedSymbolic, + spec_index: *spec_index, + edit_index: None, + }); + continue; + } + TargetRef::Peeled(local_id) => { + let (mode, reflog_message) = if local_id == remote_id { + (update::Mode::NoChangeNeeded, "TBD no change") + } else if refspecs[*spec_index].allow_non_fast_forward() { + (update::Mode::Forced, "TBD force") + } else { + todo!("check for fast-forward (is local an ancestor of remote?)") + }; + (mode, reflog_message, existing.name().to_owned()) + } } - }, + } None => (update::Mode::New, "TBD new", name.try_into()?), }; let edit = RefEdit { @@ -97,5 +113,19 @@ pub(crate) fn update( Ok(update::Outcome { edits, updates }) } +fn worktree_branches(repo: &Repository) -> Result, update::Error> { + let mut map = BTreeMap::new(); + if let Some((wt_dir, head_ref)) = repo.work_dir().zip(repo.head_ref().ok().flatten()) { + map.insert(head_ref.inner.name, wt_dir.to_owned()); + } + for proxy in repo.worktrees()? { + let repo = proxy.into_repo_with_possibly_inaccessible_worktree()?; + if let Some((wt_dir, head_ref)) = repo.work_dir().zip(repo.head_ref().ok().flatten()) { + map.insert(head_ref.inner.name, wt_dir.to_owned()); + } + } + Ok(map) +} + #[cfg(test)] mod tests; diff --git a/git-repository/src/remote/connection/fetch/update_refs/tests.rs b/git-repository/src/remote/connection/fetch/update_refs/tests.rs index d4d60bceddd..daa972ded3c 100644 --- a/git-repository/src/remote/connection/fetch/update_refs/tests.rs +++ b/git-repository/src/remote/connection/fetch/update_refs/tests.rs @@ -1,5 +1,6 @@ mod update { use crate as git; + use git_testtools::Result; fn base_repo_path() -> String { git::path::realpath( @@ -88,15 +89,16 @@ mod update { } #[test] - #[ignore] - fn checked_out_branches_in_worktrees_are_rejected_with_additional_infromation() { - let root = git_testtools::scripted_fixture_repo_read_only_with_args("make_fetch_repos.sh", [base_repo_path()]) - .unwrap(); + fn checked_out_branches_in_worktrees_are_rejected_with_additional_infromation() -> Result { + let root = git_path::realpath(git_testtools::scripted_fixture_repo_read_only_with_args( + "make_fetch_repos.sh", + [base_repo_path()], + )?)?; let repo = root.join("worktree-root"); - let repo = git::open_opts(repo, git::open::Options::isolated()).unwrap(); + let repo = git::open_opts(repo, git::open::Options::isolated())?; for (branch, path_from_root) in [ ("main", "worktree-root"), - ("wt-a-nested", "prec/wt-a"), + ("wt-a-nested", "prev/wt-a-nested"), ("wt-a", "wt-a"), ("nested-wt-b", "wt-a/nested-wt-b"), ("wt-c-locked", "wt-c-locked"), @@ -104,7 +106,7 @@ mod update { ] { let spec = format!("refs/heads/main:refs/heads/{}", branch); let (mappings, specs) = mapping_from_spec(&spec, &repo); - let out = fetch::refs::update(&repo, &mappings, &specs, fetch::DryRun::Yes).unwrap(); + let out = fetch::refs::update(&repo, &mappings, &specs, fetch::DryRun::Yes)?; assert_eq!( out.updates, @@ -117,10 +119,9 @@ mod update { }], "{}: checked-out checks are done before checking if a change would actually be required (here it isn't)", spec ); - - // TODO: add assert_eq!(out.edits.len(), 0); } + Ok(()) } #[test] diff --git a/git-repository/src/remote/connection/fetch/update_refs/update.rs b/git-repository/src/remote/connection/fetch/update_refs/update.rs index eae2ec85634..34515bbd70e 100644 --- a/git-repository/src/remote/connection/fetch/update_refs/update.rs +++ b/git-repository/src/remote/connection/fetch/update_refs/update.rs @@ -12,6 +12,10 @@ mod error { InvalidRefName(#[from] git_validate::refname::Error), #[error("Failed to update references to their new position to match their remote locations")] EditReferences(#[from] crate::reference::edit::Error), + #[error("Failed to read or iterate worktree dir")] + WorktreeListing(#[from] std::io::Error), + #[error("Could not open worktree repository")] + OpenWorktreeRepo(#[from] crate::open::Error), } } diff --git a/git-repository/src/repository/worktree.rs b/git-repository/src/repository/worktree.rs index dfe988b771f..afc572d2c34 100644 --- a/git-repository/src/repository/worktree.rs +++ b/git-repository/src/repository/worktree.rs @@ -25,21 +25,11 @@ impl crate::Repository { res.sort_by(|a, b| a.git_dir.cmp(&b.git_dir)); Ok(res) } - - /// Iterate all _linked_ worktrees in sort order and resolve them, ignoring those without an accessible work tree, into repositories - /// whose [`worktree()`][crate::Repository::worktree()] is the worktree currently being iterated. - /// - /// Note that for convenience all io errors are squelched so if there is a chance for IO errors during - /// traversal of an owned directory, better use `list()` directly. The latter allows to resolve repositories - /// even if the worktree checkout isn't accessible. - pub fn worktree_repos(&self) -> ! { - todo!() - } } /// Interact with individual worktrees and their information. impl crate::Repository { - /// Return the repository owning the main worktree. + /// Return the repository owning the main worktree, typically from a linked worktree. /// /// Note that it might be the one that is currently open if this repository doesn't point to a linked worktree. /// Also note that the main repo might be bare. From 18581d0dca277b8933885e30561d51640b88dfa4 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 29 Sep 2022 11:27:23 +0800 Subject: [PATCH 104/113] refactor (#450) Remove spec_index field as it can be derived from the corresponding mapping. --- .../src/remote/connection/fetch/update_refs/mod.rs | 13 +++---------- .../remote/connection/fetch/update_refs/tests.rs | 4 ---- git-repository/tests/remote/fetch.rs | 1 - 3 files changed, 3 insertions(+), 15 deletions(-) diff --git a/git-repository/src/remote/connection/fetch/update_refs/mod.rs b/git-repository/src/remote/connection/fetch/update_refs/mod.rs index 5c621e654f4..2dad685b376 100644 --- a/git-repository/src/remote/connection/fetch/update_refs/mod.rs +++ b/git-repository/src/remote/connection/fetch/update_refs/mod.rs @@ -16,11 +16,10 @@ pub struct Update { pub mode: update::Mode, /// The index to the edit that was created from the corresponding mapping, or `None` if there was no local ref. pub edit_index: Option, - /// The index of the ref-spec from which the source mapping originated. - pub spec_index: usize, } -/// Update all refs as derived from `mappings` and produce an `Outcome` informing about all applied changes in detail. +/// Update all refs as derived from `mappings` and produce an `Outcome` informing about all applied changes in detail, with each +/// [`update`][Update] corresponding to the [`fetch::Mapping`] of at the same index. /// If `dry_run` is true, ref transactions won't actually be applied, but are assumed to work without error so the underlying /// `repo` is not actually changed. /// @@ -51,7 +50,6 @@ pub(crate) fn update( mode: update::Mode::RejectedCurrentlyCheckedOut { worktree_dir: wt_dir.to_owned(), }, - spec_index: *spec_index, edit_index: None, }); continue; @@ -60,7 +58,6 @@ pub(crate) fn update( TargetRef::Symbolic(_) => { updates.push(Update { mode: update::Mode::RejectedSymbolic, - spec_index: *spec_index, edit_index: None, }); continue; @@ -98,11 +95,7 @@ pub(crate) fn update( } None => (update::Mode::NoChangeNeeded, None), }; - updates.push(Update { - mode, - spec_index: *spec_index, - edit_index, - }) + updates.push(Update { mode, edit_index }) } let edits = match dry_run { diff --git a/git-repository/src/remote/connection/fetch/update_refs/tests.rs b/git-repository/src/remote/connection/fetch/update_refs/tests.rs index daa972ded3c..dff45cc0ea8 100644 --- a/git-repository/src/remote/connection/fetch/update_refs/tests.rs +++ b/git-repository/src/remote/connection/fetch/update_refs/tests.rs @@ -80,7 +80,6 @@ mod update { vec![fetch::refs::Update { mode: expected_mode.clone(), edit_index: has_edit_index.then(|| 0), - spec_index: 0 }], "{spec:?}: {detail}" ); @@ -115,7 +114,6 @@ mod update { worktree_dir: root.join(path_from_root), }, edit_index: None, - spec_index: 0, }], "{}: checked-out checks are done before checking if a change would actually be required (here it isn't)", spec ); @@ -135,7 +133,6 @@ mod update { vec![fetch::refs::Update { mode: fetch::refs::update::Mode::RejectedSymbolic, edit_index: None, - spec_index: 0, }], "this also protects from writing HEAD, which should in theory be impossible to get from a refspec as it normalizes partial ref names" ); @@ -154,7 +151,6 @@ mod update { vec![fetch::refs::Update { mode: fetch::refs::update::Mode::RejectedNonFastForward, edit_index: Some(0), - spec_index: 0, }] ); assert_eq!(out.edits.len(), 1); diff --git a/git-repository/tests/remote/fetch.rs b/git-repository/tests/remote/fetch.rs index c1530f77f89..449edbe12e6 100644 --- a/git-repository/tests/remote/fetch.rs +++ b/git-repository/tests/remote/fetch.rs @@ -85,7 +85,6 @@ mod blocking_io { vec![fetch::refs::Update { mode: fetch::refs::update::Mode::New, edit_index: Some(0), - spec_index: 0 }] ); for (_update, mapping, _spec, edit) in refs.iter_mapping_updates( From ba41b6c10c073926802e38a8fa035df8444affef Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 29 Sep 2022 16:31:41 +0800 Subject: [PATCH 105/113] no-clobber special case for tags (#450) --- .../connection/fetch/update_refs/mod.rs | 29 ++++++++++++------- .../connection/fetch/update_refs/tests.rs | 13 ++++++++- .../connection/fetch/update_refs/update.rs | 3 ++ 3 files changed, 33 insertions(+), 12 deletions(-) diff --git a/git-repository/src/remote/connection/fetch/update_refs/mod.rs b/git-repository/src/remote/connection/fetch/update_refs/mod.rs index 2dad685b376..364c6038882 100644 --- a/git-repository/src/remote/connection/fetch/update_refs/mod.rs +++ b/git-repository/src/remote/connection/fetch/update_refs/mod.rs @@ -1,4 +1,5 @@ use crate::remote::fetch; +use crate::remote::fetch::refs::update::Mode; use crate::Repository; use git_ref::transaction::{Change, LogChange, PreviousValue, RefEdit, RefLog}; use git_ref::{Target, TargetRef}; @@ -18,6 +19,12 @@ pub struct Update { pub edit_index: Option, } +impl From for Update { + fn from(mode: Mode) -> Self { + Update { mode, edit_index: None } + } +} + /// Update all refs as derived from `mappings` and produce an `Outcome` informing about all applied changes in detail, with each /// [`update`][Update] corresponding to the [`fetch::Mapping`] of at the same index. /// If `dry_run` is true, ref transactions won't actually be applied, but are assumed to work without error so the underlying @@ -46,20 +53,15 @@ pub(crate) fn update( let (mode, reflog_message, name) = match repo.try_find_reference(name)? { Some(existing) => { if let Some(wt_dir) = checked_out_branches.get(existing.name()) { - updates.push(Update { - mode: update::Mode::RejectedCurrentlyCheckedOut { - worktree_dir: wt_dir.to_owned(), - }, - edit_index: None, - }); + let mode = update::Mode::RejectedCurrentlyCheckedOut { + worktree_dir: wt_dir.to_owned(), + }; + updates.push(mode.into()); continue; } match existing.target() { TargetRef::Symbolic(_) => { - updates.push(Update { - mode: update::Mode::RejectedSymbolic, - edit_index: None, - }); + updates.push(update::Mode::RejectedSymbolic.into()); continue; } TargetRef::Peeled(local_id) => { @@ -68,7 +70,12 @@ pub(crate) fn update( } else if refspecs[*spec_index].allow_non_fast_forward() { (update::Mode::Forced, "TBD force") } else { - todo!("check for fast-forward (is local an ancestor of remote?)") + if let Some(git_ref::Category::Tag) = existing.name().category() { + updates.push(update::Mode::RejectedTagUpdate.into()); + continue; + } else { + todo!("check for fast-forward (is local an ancestor of remote?)") + } }; (mode, reflog_message, existing.name().to_owned()) } diff --git a/git-repository/src/remote/connection/fetch/update_refs/tests.rs b/git-repository/src/remote/connection/fetch/update_refs/tests.rs index dff45cc0ea8..17f75a1c868 100644 --- a/git-repository/src/remote/connection/fetch/update_refs/tests.rs +++ b/git-repository/src/remote/connection/fetch/update_refs/tests.rs @@ -22,7 +22,6 @@ mod update { use git_ref::TargetRef; #[test] - #[ignore] fn various_valid_updates() { let repo = repo("two-origins"); // TODO: test reflog message (various cases if it's new) @@ -57,6 +56,18 @@ mod update { true, "a forced non-fastforward (main goes backwards)", ), + ( + "+refs/heads/main:refs/tags/b-tag", + fetch::refs::update::Mode::Forced, + true, + "tags can only be forced", + ), + ( + "refs/heads/main:refs/tags/b-tag", + fetch::refs::update::Mode::RejectedTagUpdate, + false, + "otherwise a tag is always refusing itself to be overwritten (no-clobber)", + ), ( "+refs/remotes/origin/g:refs/heads/main", fetch::refs::update::Mode::RejectedCurrentlyCheckedOut { diff --git a/git-repository/src/remote/connection/fetch/update_refs/update.rs b/git-repository/src/remote/connection/fetch/update_refs/update.rs index 34515bbd70e..031263d0b73 100644 --- a/git-repository/src/remote/connection/fetch/update_refs/update.rs +++ b/git-repository/src/remote/connection/fetch/update_refs/update.rs @@ -41,6 +41,9 @@ pub enum Mode { Forced, /// A new ref has been created as there was none before. New, + /// Tags can never be overwritten (whether the new object would be a fast-forward or not, or unchanged), unless the refspec + /// specifies force. + RejectedTagUpdate, /// The reference update would not have been a fast-forward, and force is not specified in the ref-spec. RejectedNonFastForward, /// The update of a local symbolic reference was rejected. From d79a7a67ef77ab9d6dc871e16d535a509eb78e49 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 29 Sep 2022 16:56:45 +0800 Subject: [PATCH 106/113] assure objects exist before setting them. (#450) --- .../connection/fetch/update_refs/mod.rs | 5 +++++ .../connection/fetch/update_refs/tests.rs | 20 +++++++++++++++---- .../connection/fetch/update_refs/update.rs | 11 +++++++--- 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/git-repository/src/remote/connection/fetch/update_refs/mod.rs b/git-repository/src/remote/connection/fetch/update_refs/mod.rs index 364c6038882..783e40d2330 100644 --- a/git-repository/src/remote/connection/fetch/update_refs/mod.rs +++ b/git-repository/src/remote/connection/fetch/update_refs/mod.rs @@ -1,6 +1,7 @@ use crate::remote::fetch; use crate::remote::fetch::refs::update::Mode; use crate::Repository; +use git_pack::Find; use git_ref::transaction::{Change, LogChange, PreviousValue, RefEdit, RefLog}; use git_ref::{Target, TargetRef}; use std::collections::BTreeMap; @@ -47,6 +48,10 @@ pub(crate) fn update( } in mappings { let remote_id = remote.as_id(); + if !repo.objects.contains(remote_id) { + updates.push(update::Mode::RejectedSourceObjectNotFound { id: remote_id.into() }.into()); + continue; + } let checked_out_branches = worktree_branches(repo)?; let (mode, edit_index) = match local { Some(name) => { diff --git a/git-repository/src/remote/connection/fetch/update_refs/tests.rs b/git-repository/src/remote/connection/fetch/update_refs/tests.rs index 17f75a1c868..4bd44fbabfa 100644 --- a/git-repository/src/remote/connection/fetch/update_refs/tests.rs +++ b/git-repository/src/remote/connection/fetch/update_refs/tests.rs @@ -1,6 +1,6 @@ mod update { use crate as git; - use git_testtools::Result; + use git_testtools::{hex_to_id, Result}; fn base_repo_path() -> String { git::path::realpath( @@ -76,6 +76,14 @@ mod update { false, "checked out branches cannot be written, as it requires a merge of sorts which isn't done here", ), + ( + "ffffffffffffffffffffffffffffffffffffffff:refs/heads/invalid-source-object", + fetch::refs::update::Mode::RejectedSourceObjectNotFound { + id: hex_to_id("ffffffffffffffffffffffffffffffffffffffff"), + }, + false, + "checked out branches cannot be written, as it requires a merge of sorts which isn't done here", + ), // ( // TODO: make fast-forwards work // "refs/remotes/origin/g:refs/heads/not-currently-checked-out", // fetch::refs::update::Mode::FastForward, @@ -177,9 +185,13 @@ mod update { .mappings .into_iter() .map(|m| fetch::Mapping { - remote: fetch::Source::Ref( - references[m.item_index.expect("set as all items are backed by ref")].clone(), - ), + remote: m + .item_index + .map(|idx| fetch::Source::Ref(references[idx].clone())) + .unwrap_or_else(|| match m.lhs { + git_refspec::match_group::SourceRef::ObjectId(id) => fetch::Source::ObjectId(id), + _ => unreachable!("not a ref, must be id: {:?}", m), + }), local: m.rhs.map(|r| r.into_owned()), spec_index: m.spec_index, }) diff --git a/git-repository/src/remote/connection/fetch/update_refs/update.rs b/git-repository/src/remote/connection/fetch/update_refs/update.rs index 031263d0b73..a0658f934be 100644 --- a/git-repository/src/remote/connection/fetch/update_refs/update.rs +++ b/git-repository/src/remote/connection/fetch/update_refs/update.rs @@ -35,12 +35,20 @@ pub struct Outcome { /// Describe the way a ref was updated #[derive(Debug, Clone, PartialEq, Eq)] pub enum Mode { + /// No change was attempted as the remote ref didn't change compared to the current ref, or because no remote ref was specified + /// in the ref-spec. + NoChangeNeeded, /// The old ref's commit was an ancestor of the new one, allowing for a fast-forward without a merge. FastForward, /// The ref was set to point to the new commit from the remote without taking into consideration its ancestry. Forced, /// A new ref has been created as there was none before. New, + /// The object id to set the target reference to could not be found. + RejectedSourceObjectNotFound { + /// The id of the object that didn't exist in the object database, even though it should since it should be part of the pack. + id: git_hash::ObjectId, + }, /// Tags can never be overwritten (whether the new object would be a fast-forward or not, or unchanged), unless the refspec /// specifies force. RejectedTagUpdate, @@ -55,9 +63,6 @@ pub enum Mode { /// The path to the worktree directory where the branch is checked out. worktree_dir: PathBuf, }, - /// No change was attempted as the remote ref didn't change compared to the current ref, or because no remote ref was specified - /// in the ref-spec. - NoChangeNeeded, } impl Outcome { From b521748974c6cb021121cdcc3bbbca5b80987336 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 29 Sep 2022 16:57:59 +0800 Subject: [PATCH 107/113] thanks clippy --- .../src/remote/connection/fetch/update_refs/mod.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/git-repository/src/remote/connection/fetch/update_refs/mod.rs b/git-repository/src/remote/connection/fetch/update_refs/mod.rs index 783e40d2330..13830c44445 100644 --- a/git-repository/src/remote/connection/fetch/update_refs/mod.rs +++ b/git-repository/src/remote/connection/fetch/update_refs/mod.rs @@ -74,13 +74,11 @@ pub(crate) fn update( (update::Mode::NoChangeNeeded, "TBD no change") } else if refspecs[*spec_index].allow_non_fast_forward() { (update::Mode::Forced, "TBD force") + } else if let Some(git_ref::Category::Tag) = existing.name().category() { + updates.push(update::Mode::RejectedTagUpdate.into()); + continue; } else { - if let Some(git_ref::Category::Tag) = existing.name().category() { - updates.push(update::Mode::RejectedTagUpdate.into()); - continue; - } else { - todo!("check for fast-forward (is local an ancestor of remote?)") - } + todo!("check for fast-forward (is local an ancestor of remote?)") }; (mode, reflog_message, existing.name().to_owned()) } From 91859d5140a45874d8d934773a47eee6b6a5a126 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 29 Sep 2022 18:48:43 +0800 Subject: [PATCH 108/113] facilities to test reflog messages (#450) --- .../connection/fetch/update_refs/mod.rs | 6 +-- .../connection/fetch/update_refs/tests.rs | 37 +++++++++++++------ 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/git-repository/src/remote/connection/fetch/update_refs/mod.rs b/git-repository/src/remote/connection/fetch/update_refs/mod.rs index 13830c44445..b19bb209e82 100644 --- a/git-repository/src/remote/connection/fetch/update_refs/mod.rs +++ b/git-repository/src/remote/connection/fetch/update_refs/mod.rs @@ -71,9 +71,9 @@ pub(crate) fn update( } TargetRef::Peeled(local_id) => { let (mode, reflog_message) = if local_id == remote_id { - (update::Mode::NoChangeNeeded, "TBD no change") + (update::Mode::NoChangeNeeded, "TBD") } else if refspecs[*spec_index].allow_non_fast_forward() { - (update::Mode::Forced, "TBD force") + (update::Mode::Forced, "TBD") } else if let Some(git_ref::Category::Tag) = existing.name().category() { updates.push(update::Mode::RejectedTagUpdate.into()); continue; @@ -84,7 +84,7 @@ pub(crate) fn update( } } } - None => (update::Mode::New, "TBD new", name.try_into()?), + None => (update::Mode::New, "TBD", name.try_into()?), }; let edit = RefEdit { change: Change::Update { diff --git a/git-repository/src/remote/connection/fetch/update_refs/tests.rs b/git-repository/src/remote/connection/fetch/update_refs/tests.rs index 4bd44fbabfa..387138b4cfe 100644 --- a/git-repository/src/remote/connection/fetch/update_refs/tests.rs +++ b/git-repository/src/remote/connection/fetch/update_refs/tests.rs @@ -19,53 +19,54 @@ mod update { git::open_opts(dir.join(name), git::open::Options::isolated()).unwrap() } use crate::remote::fetch; + use git_ref::transaction::Change; use git_ref::TargetRef; #[test] fn various_valid_updates() { let repo = repo("two-origins"); // TODO: test reflog message (various cases if it's new) - for (spec, expected_mode, has_edit_index, detail) in [ + for (spec, expected_mode, reflog_message, detail) in [ ( "refs/heads/main:refs/remotes/origin/main", fetch::refs::update::Mode::NoChangeNeeded, - true, + Some("TBD"), "these refs are en-par since the initial clone", ), ( "refs/heads/main", fetch::refs::update::Mode::NoChangeNeeded, - false, + None, "without local destination ref there is nothing to do for us, ever (except for FETCH_HEADs) later", ), ( "refs/heads/main:refs/remotes/origin/new-main", fetch::refs::update::Mode::New, - true, + Some("TBD"), "the destination branch doesn't exist and needs to be created", ), ( "refs/heads/main:refs/remotes/origin/new-main", fetch::refs::update::Mode::New, - true, + Some("TBD"), "just to validate that we really are in dry-run mode, or else this ref would be present now", ), ( "+refs/heads/main:refs/remotes/origin/g", fetch::refs::update::Mode::Forced, - true, + Some("TBD"), "a forced non-fastforward (main goes backwards)", ), ( "+refs/heads/main:refs/tags/b-tag", fetch::refs::update::Mode::Forced, - true, + Some("TBD"), "tags can only be forced", ), ( "refs/heads/main:refs/tags/b-tag", fetch::refs::update::Mode::RejectedTagUpdate, - false, + None, "otherwise a tag is always refusing itself to be overwritten (no-clobber)", ), ( @@ -73,7 +74,7 @@ mod update { fetch::refs::update::Mode::RejectedCurrentlyCheckedOut { worktree_dir: repo.work_dir().expect("present").to_owned(), }, - false, + None, "checked out branches cannot be written, as it requires a merge of sorts which isn't done here", ), ( @@ -81,7 +82,7 @@ mod update { fetch::refs::update::Mode::RejectedSourceObjectNotFound { id: hex_to_id("ffffffffffffffffffffffffffffffffffffffff"), }, - false, + None, "checked out branches cannot be written, as it requires a merge of sorts which isn't done here", ), // ( // TODO: make fast-forwards work @@ -98,11 +99,23 @@ mod update { out.updates, vec![fetch::refs::Update { mode: expected_mode.clone(), - edit_index: has_edit_index.then(|| 0), + edit_index: reflog_message.map(|_| 0), }], "{spec:?}: {detail}" ); - assert_eq!(out.edits.len(), has_edit_index.then(|| 1).unwrap_or(0)); + assert_eq!(out.edits.len(), reflog_message.map(|_| 1).unwrap_or(0)); + if let Some(reflog_message) = reflog_message { + let edit = &out.edits[0]; + match &edit.change { + Change::Update { log, .. } => { + assert_eq!( + log.message, reflog_message, + "reflog messages are specific and we emulate git word for word" + ); + } + _ => unreachable!("only updates"), + } + } } } From 9a072bd2e495b19c479e831f62172482da511d8c Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 29 Sep 2022 21:08:42 +0800 Subject: [PATCH 109/113] refactor (#450) --- .../tests/repository/config/config_snapshot/mod.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/git-repository/tests/repository/config/config_snapshot/mod.rs b/git-repository/tests/repository/config/config_snapshot/mod.rs index cf46f53f5a4..98e1053fe61 100644 --- a/git-repository/tests/repository/config/config_snapshot/mod.rs +++ b/git-repository/tests/repository/config/config_snapshot/mod.rs @@ -6,18 +6,18 @@ fn commit_auto_rollback() -> crate::Result { assert_eq!(repo.head_id()?.shorten()?.to_string(), "3189cd3"); { - let mut repo = repo.config_snapshot_mut(); - repo.set_raw_value("core", None, "abbrev", "4")?; - let repo = repo.commit_auto_rollback()?; + let mut config = repo.config_snapshot_mut(); + config.set_raw_value("core", None, "abbrev", "4")?; + let repo = config.commit_auto_rollback()?; assert_eq!(repo.head_id()?.shorten()?.to_string(), "3189"); } assert_eq!(repo.head_id()?.shorten()?.to_string(), "3189cd3"); let repo = { - let mut repo = repo.config_snapshot_mut(); - repo.set_raw_value("core", None, "abbrev", "4")?; - let repo = repo.commit_auto_rollback()?; + let mut config = repo.config_snapshot_mut(); + config.set_raw_value("core", None, "abbrev", "4")?; + let repo = config.commit_auto_rollback()?; assert_eq!(repo.head_id()?.shorten()?.to_string(), "3189"); repo.rollback()? }; From 6ebbbc153dcf4eedb2afbd561c4d2ce342f6289b Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 30 Sep 2022 09:58:37 +0800 Subject: [PATCH 110/113] support reflog message prefix (#450) --- git-repository/src/remote/connection/fetch/mod.rs | 1 + .../src/remote/connection/fetch/update_refs/mod.rs | 4 +++- .../src/remote/connection/fetch/update_refs/tests.rs | 11 ++++++----- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/git-repository/src/remote/connection/fetch/mod.rs b/git-repository/src/remote/connection/fetch/mod.rs index 41381169ee9..3fffb30d933 100644 --- a/git-repository/src/remote/connection/fetch/mod.rs +++ b/git-repository/src/remote/connection/fetch/mod.rs @@ -177,6 +177,7 @@ where let update_refs = refs::update( repo, + "fetch", &self.ref_map.mappings, con.remote.refspecs(crate::remote::Direction::Fetch), fetch::DryRun::No, diff --git a/git-repository/src/remote/connection/fetch/update_refs/mod.rs b/git-repository/src/remote/connection/fetch/update_refs/mod.rs index b19bb209e82..8a6f019a9bf 100644 --- a/git-repository/src/remote/connection/fetch/update_refs/mod.rs +++ b/git-repository/src/remote/connection/fetch/update_refs/mod.rs @@ -30,10 +30,12 @@ impl From for Update { /// [`update`][Update] corresponding to the [`fetch::Mapping`] of at the same index. /// If `dry_run` is true, ref transactions won't actually be applied, but are assumed to work without error so the underlying /// `repo` is not actually changed. +/// `action` is the prefix used for reflog entries, and is typically "fetch". /// /// It can be used to produce typical information that one is used to from `git fetch`. pub(crate) fn update( repo: &Repository, + action: &str, mappings: &[fetch::Mapping], refspecs: &[git_refspec::RefSpec], dry_run: fetch::DryRun, @@ -91,7 +93,7 @@ pub(crate) fn update( log: LogChange { mode: RefLog::AndReference, force_create_reflog: false, - message: reflog_message.into(), + message: format!("{}: {}", action, reflog_message).into(), }, expected: PreviousValue::ExistingMustMatch(Target::Peeled(remote_id.into())), new: Target::Peeled(remote_id.into()), diff --git a/git-repository/src/remote/connection/fetch/update_refs/tests.rs b/git-repository/src/remote/connection/fetch/update_refs/tests.rs index 387138b4cfe..534c1757a25 100644 --- a/git-repository/src/remote/connection/fetch/update_refs/tests.rs +++ b/git-repository/src/remote/connection/fetch/update_refs/tests.rs @@ -93,7 +93,7 @@ mod update { // ), ] { let (mapping, specs) = mapping_from_spec(spec, &repo); - let out = fetch::refs::update(&repo, &mapping, &specs, fetch::DryRun::Yes).unwrap(); + let out = fetch::refs::update(&repo, "action", &mapping, &specs, fetch::DryRun::Yes).unwrap(); assert_eq!( out.updates, @@ -109,7 +109,8 @@ mod update { match &edit.change { Change::Update { log, .. } => { assert_eq!( - log.message, reflog_message, + log.message, + format!("action: {}", reflog_message), "reflog messages are specific and we emulate git word for word" ); } @@ -137,7 +138,7 @@ mod update { ] { let spec = format!("refs/heads/main:refs/heads/{}", branch); let (mappings, specs) = mapping_from_spec(&spec, &repo); - let out = fetch::refs::update(&repo, &mappings, &specs, fetch::DryRun::Yes)?; + let out = fetch::refs::update(&repo, "action", &mappings, &specs, fetch::DryRun::Yes)?; assert_eq!( out.updates, @@ -158,7 +159,7 @@ mod update { fn symbolic_refs_are_never_written() { let repo = repo("two-origins"); let (mappings, specs) = mapping_from_spec("refs/heads/main:refs/heads/symbolic", &repo); - let out = fetch::refs::update(&repo, &mappings, &specs, fetch::DryRun::Yes).unwrap(); + let out = fetch::refs::update(&repo, "action", &mappings, &specs, fetch::DryRun::Yes).unwrap(); assert_eq!( out.updates, @@ -176,7 +177,7 @@ mod update { fn fast_forward_is_not_implemented_yet_but_should_be_denied() { let repo = repo("two-origins"); let (mappings, specs) = mapping_from_spec("refs/heads/main:refs/remotes/origin/g", &repo); - let out = fetch::refs::update(&repo, &mappings, &specs, fetch::DryRun::Yes).unwrap(); + let out = fetch::refs::update(&repo, "action", &mappings, &specs, fetch::DryRun::Yes).unwrap(); assert_eq!( out.updates, From 2a76908f3ebe846e96b783075bb5395d0bb9aaa0 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 30 Sep 2022 10:25:54 +0800 Subject: [PATCH 111/113] test all reflog messages that are expected, sans fast-forward (#450) --- .../connection/fetch/update_refs/mod.rs | 18 ++++++++++--- .../connection/fetch/update_refs/tests.rs | 27 ++++++++++++++----- 2 files changed, 35 insertions(+), 10 deletions(-) diff --git a/git-repository/src/remote/connection/fetch/update_refs/mod.rs b/git-repository/src/remote/connection/fetch/update_refs/mod.rs index 8a6f019a9bf..00ed5e6a4db 100644 --- a/git-repository/src/remote/connection/fetch/update_refs/mod.rs +++ b/git-repository/src/remote/connection/fetch/update_refs/mod.rs @@ -73,9 +73,13 @@ pub(crate) fn update( } TargetRef::Peeled(local_id) => { let (mode, reflog_message) = if local_id == remote_id { - (update::Mode::NoChangeNeeded, "TBD") + (update::Mode::NoChangeNeeded, "no update will be performed") } else if refspecs[*spec_index].allow_non_fast_forward() { - (update::Mode::Forced, "TBD") + let reflog_msg = match existing.name().category() { + Some(git_ref::Category::Tag) => "updating tag", + _ => "forced-update", + }; + (update::Mode::Forced, reflog_msg) } else if let Some(git_ref::Category::Tag) = existing.name().category() { updates.push(update::Mode::RejectedTagUpdate.into()); continue; @@ -86,7 +90,15 @@ pub(crate) fn update( } } } - None => (update::Mode::New, "TBD", name.try_into()?), + None => { + let name: git_ref::FullName = name.try_into()?; + let reflog_msg = match name.category() { + Some(git_ref::Category::Tag) => "storing tag", + Some(git_ref::Category::LocalBranch) => "storing head", + _ => "storing ref", + }; + (update::Mode::New, reflog_msg, name) + } }; let edit = RefEdit { change: Change::Update { diff --git a/git-repository/src/remote/connection/fetch/update_refs/tests.rs b/git-repository/src/remote/connection/fetch/update_refs/tests.rs index 534c1757a25..784eb9551d5 100644 --- a/git-repository/src/remote/connection/fetch/update_refs/tests.rs +++ b/git-repository/src/remote/connection/fetch/update_refs/tests.rs @@ -30,7 +30,7 @@ mod update { ( "refs/heads/main:refs/remotes/origin/main", fetch::refs::update::Mode::NoChangeNeeded, - Some("TBD"), + Some("no update will be performed"), "these refs are en-par since the initial clone", ), ( @@ -42,25 +42,37 @@ mod update { ( "refs/heads/main:refs/remotes/origin/new-main", fetch::refs::update::Mode::New, - Some("TBD"), + Some("storing ref"), "the destination branch doesn't exist and needs to be created", ), ( - "refs/heads/main:refs/remotes/origin/new-main", + "refs/heads/main:refs/heads/feature", + fetch::refs::update::Mode::New, + Some("storing head"), + "reflog messages are specific to the type of branch stored, to some limited extend", + ), + ( + "refs/heads/main:refs/tags/new-tag", + fetch::refs::update::Mode::New, + Some("storing tag"), + "reflog messages are specific to the type of branch stored, to some limited extend", + ), + ( + "+refs/heads/main:refs/remotes/origin/new-main", fetch::refs::update::Mode::New, - Some("TBD"), + Some("storing ref"), "just to validate that we really are in dry-run mode, or else this ref would be present now", ), ( "+refs/heads/main:refs/remotes/origin/g", fetch::refs::update::Mode::Forced, - Some("TBD"), + Some("forced-update"), "a forced non-fastforward (main goes backwards)", ), ( "+refs/heads/main:refs/tags/b-tag", fetch::refs::update::Mode::Forced, - Some("TBD"), + Some("updating tag"), "tags can only be forced", ), ( @@ -111,7 +123,8 @@ mod update { assert_eq!( log.message, format!("action: {}", reflog_message), - "reflog messages are specific and we emulate git word for word" + "{}: reflog messages are specific and we emulate git word for word", + spec ); } _ => unreachable!("only updates"), From 370ed3dcc393eca7a393ea0150f698a9fc844320 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 30 Sep 2022 14:15:23 +0800 Subject: [PATCH 112/113] feat: `transaction::Change::new_value()` to get easy access to new values of references. (#450) That's more convenient than matching on the enum. --- git-ref/src/transaction/mod.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/git-ref/src/transaction/mod.rs b/git-ref/src/transaction/mod.rs index e4fa518e756..5eeec36c867 100644 --- a/git-ref/src/transaction/mod.rs +++ b/git-ref/src/transaction/mod.rs @@ -83,7 +83,15 @@ pub enum Change { } impl Change { - /// Return references to values that are in common between all variants. + /// Return references to values that are the new value after the change is applied, if this is an update. + pub fn new_value(&self) -> Option> { + match self { + Change::Update { new, .. } => new.to_ref().into(), + Change::Delete { .. } => None, + } + } + + /// Return references to values that are in common between all variants and denote the previous observed value. pub fn previous_value(&self) -> Option> { match self { // TODO: use or-patterns once MRV is larger than 1.52 (and this is supported) From ef9fa9835f9abaaa5034d62359e2899c0dd51408 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Fri, 30 Sep 2022 14:16:10 +0800 Subject: [PATCH 113/113] dry-run mode for fetch (#450) --- .../src/remote/connection/fetch/mod.rs | 64 ++++++++++++++----- .../connection/fetch/update_refs/mod.rs | 5 +- .../connection/fetch/update_refs/tests.rs | 9 ++- git-repository/src/remote/fetch.rs | 1 - git-repository/tests/remote/fetch.rs | 56 ++++++++++------ 5 files changed, 93 insertions(+), 42 deletions(-) diff --git a/git-repository/src/remote/connection/fetch/mod.rs b/git-repository/src/remote/connection/fetch/mod.rs index 3fffb30d933..4cf38ba47ca 100644 --- a/git-repository/src/remote/connection/fetch/mod.rs +++ b/git-repository/src/remote/connection/fetch/mod.rs @@ -1,4 +1,4 @@ -use crate::remote::fetch::RefMap; +use crate::remote::fetch::{DryRun, RefMap}; use crate::remote::{fetch, ref_map, Connection}; use crate::Progress; use git_odb::FindExt; @@ -42,6 +42,12 @@ pub enum Status { /// Information collected while updating references. update_refs: refs::update::Outcome, }, + /// A dry run was performed which leaves the local repository without any change + /// nor will a pack have been received. + DryRun { + /// Information about what updates to refs would have been done. + update_refs: refs::update::Outcome, + }, } /// The outcome of receiving a pack via [`Prepare::receive()`]. @@ -77,6 +83,7 @@ where Ok(Prepare { con: Some(self), ref_map, + dry_run: fetch::DryRun::No, }) } } @@ -159,17 +166,23 @@ where iteration_mode: git_pack::data::input::Mode::Verify, object_hash: con.remote.repo.object_hash(), }; - let write_pack_bundle = git_pack::Bundle::write_to_directory( - reader, - Some(repo.objects.store_ref().path().join("pack")), - con.progress, - should_interrupt, - Some(Box::new({ - let repo = repo.clone(); - move |oid, buf| repo.objects.find(oid, buf).ok() - })), - options, - )?; + + let write_pack_bundle = if matches!(self.dry_run, fetch::DryRun::No) { + Some(git_pack::Bundle::write_to_directory( + reader, + Some(repo.objects.store_ref().path().join("pack")), + con.progress, + should_interrupt, + Some(Box::new({ + let repo = repo.clone(); + move |oid, buf| repo.objects.find(oid, buf).ok() + })), + options, + )?) + } else { + drop(reader); + None + }; if matches!(protocol_version, git_protocol::transport::Protocol::V2) { git_protocol::fetch::indicate_end_of_interaction(&mut con.transport).ok(); @@ -180,14 +193,17 @@ where "fetch", &self.ref_map.mappings, con.remote.refspecs(crate::remote::Direction::Fetch), - fetch::DryRun::No, + self.dry_run, )?; Ok(Outcome { ref_map: std::mem::take(&mut self.ref_map), - status: Status::Change { - write_pack_bundle, - update_refs, + status: match write_pack_bundle { + Some(write_pack_bundle) => Status::Change { + write_pack_bundle, + update_refs, + }, + None => Status::DryRun { update_refs }, }, }) } @@ -212,13 +228,27 @@ mod config; pub mod refs; /// A structure to hold the result of the handshake with the remote and configure the upcoming fetch operation. -#[allow(dead_code)] pub struct Prepare<'remote, 'repo, T, P> where T: Transport, { con: Option>, ref_map: RefMap<'remote>, + dry_run: fetch::DryRun, +} + +/// Builder +impl<'remote, 'repo, T, P> Prepare<'remote, 'repo, T, P> +where + T: Transport, +{ + /// If dry run is enabled, no change to the repository will be made. + /// + /// This works by not actually fetching the pack after negotiating it, nor will refs be updated. + pub fn with_dry_run(mut self, enabled: bool) -> Self { + self.dry_run = enabled.then(|| fetch::DryRun::Yes).unwrap_or(DryRun::No); + self + } } impl<'remote, 'repo, T, P> Drop for Prepare<'remote, 'repo, T, P> diff --git a/git-repository/src/remote/connection/fetch/update_refs/mod.rs b/git-repository/src/remote/connection/fetch/update_refs/mod.rs index 00ed5e6a4db..9cfbd71aaae 100644 --- a/git-repository/src/remote/connection/fetch/update_refs/mod.rs +++ b/git-repository/src/remote/connection/fetch/update_refs/mod.rs @@ -29,7 +29,8 @@ impl From for Update { /// Update all refs as derived from `mappings` and produce an `Outcome` informing about all applied changes in detail, with each /// [`update`][Update] corresponding to the [`fetch::Mapping`] of at the same index. /// If `dry_run` is true, ref transactions won't actually be applied, but are assumed to work without error so the underlying -/// `repo` is not actually changed. +/// `repo` is not actually changed. Also it won't perform an 'object exists' check as these are likely not to exist as the pack +/// wasn't fetched either. /// `action` is the prefix used for reflog entries, and is typically "fetch". /// /// It can be used to produce typical information that one is used to from `git fetch`. @@ -50,7 +51,7 @@ pub(crate) fn update( } in mappings { let remote_id = remote.as_id(); - if !repo.objects.contains(remote_id) { + if dry_run == fetch::DryRun::No && !repo.objects.contains(remote_id) { updates.push(update::Mode::RejectedSourceObjectNotFound { id: remote_id.into() }.into()); continue; } diff --git a/git-repository/src/remote/connection/fetch/update_refs/tests.rs b/git-repository/src/remote/connection/fetch/update_refs/tests.rs index 784eb9551d5..ddbba5f11f9 100644 --- a/git-repository/src/remote/connection/fetch/update_refs/tests.rs +++ b/git-repository/src/remote/connection/fetch/update_refs/tests.rs @@ -105,7 +105,14 @@ mod update { // ), ] { let (mapping, specs) = mapping_from_spec(spec, &repo); - let out = fetch::refs::update(&repo, "action", &mapping, &specs, fetch::DryRun::Yes).unwrap(); + let out = fetch::refs::update( + &repo, + "action", + &mapping, + &specs, + reflog_message.map(|_| fetch::DryRun::Yes).unwrap_or(fetch::DryRun::No), + ) + .unwrap(); assert_eq!( out.updates, diff --git a/git-repository/src/remote/fetch.rs b/git-repository/src/remote/fetch.rs index 588576ca74e..39410c6746a 100644 --- a/git-repository/src/remote/fetch.rs +++ b/git-repository/src/remote/fetch.rs @@ -2,7 +2,6 @@ use crate::bstr::BString; /// If `Yes`, don't really make changes but do as much as possible to get an idea of what would be done. #[derive(Debug, Copy, Clone, PartialEq, Eq)] -#[cfg_attr(not(test), allow(dead_code))] #[cfg(feature = "blocking-network-client")] pub(crate) enum DryRun { /// Enable dry-run mode and don't actually change the underlying repository in any way. diff --git a/git-repository/tests/remote/fetch.rs b/git-repository/tests/remote/fetch.rs index 449edbe12e6..08f8248d45f 100644 --- a/git-repository/tests/remote/fetch.rs +++ b/git-repository/tests/remote/fetch.rs @@ -55,16 +55,17 @@ mod blocking_io { } // Some updates to be fetched - { + for dry_run in [true, false] { let remote = repo.find_remote("changes-on-top-of-origin")?; let outcome: git::remote::fetch::Outcome = remote .connect(Fetch, progress::Discard)? .prepare_fetch(Default::default())? + .with_dry_run(dry_run) .receive(&AtomicBool::default())?; - match outcome.status { + let refs = match outcome.status { fetch::Status::Change { write_pack_bundle, - update_refs: refs, + update_refs, } => { assert_eq!(write_pack_bundle.pack_kind, git::odb::pack::data::Version::V2); assert_eq!(write_pack_bundle.object_hash, repo.object_hash()); @@ -80,27 +81,40 @@ mod blocking_io { assert!(write_pack_bundle.data_path.map_or(false, |f| f.is_file())); assert!(write_pack_bundle.index_path.map_or(false, |f| f.is_file())); + update_refs + } + fetch::Status::DryRun { update_refs } => update_refs, + fetch::Status::NoChange => unreachable!("we firmly expect changes here"), + }; + + assert_eq!( + refs.updates, + vec![fetch::refs::Update { + mode: fetch::refs::update::Mode::New, + edit_index: Some(0), + }] + ); + for (_update, mapping, _spec, edit) in + refs.iter_mapping_updates(&outcome.ref_map.mappings, remote.refspecs(Fetch)) + { + let edit = edit.expect("refedit present even if it's a no-op"); + if dry_run { assert_eq!( - refs.updates, - vec![fetch::refs::Update { - mode: fetch::refs::update::Mode::New, - edit_index: Some(0), - }] + edit.change.new_value().expect("no deletions").id(), + mapping.remote.as_id() + ); + assert!( + repo.try_find_reference(edit.name.as_ref())?.is_none(), + "no ref created in dry-run mode" + ); + } else { + let r = repo.find_reference(edit.name.as_ref()).unwrap(); + assert_eq!( + r.id(), + *mapping.remote.as_id(), + "local reference should point to remote id" ); - for (_update, mapping, _spec, edit) in refs.iter_mapping_updates( - &outcome.ref_map.mappings, - remote.refspecs(git::remote::Direction::Fetch), - ) { - let edit = edit.expect("refedit present even if it's a no-op"); - let r = repo.find_reference(edit.name.as_ref()).unwrap(); - assert_eq!( - r.id(), - *mapping.remote.as_id(), - "local reference should point to remote id" - ); - } } - fetch::Status::NoChange => unreachable!("we firmly expect changes here"), } } }