fetch pack

Cargo.toml

@@ -92,6 +92,10 @@ env_logger = { version = "0.9.0", default-features = false }
 crosstermion = { version = "0.10.1", optional = true, default-features = false }
 futures-lite = { version = "1.12.0", optional = true, default-features = false, features = ["std"] }
 
+# for progress
+owo-colors = "3.5.0"
+tabled = { version = "0.8.0", default-features = false }
+
 document-features = { version = "0.2.0", optional = true }
 
 [profile.dev.package]

DEVELOPMENT.md

@@ -18,6 +18,21 @@
    * We `thiserror` generally.
  * Adhere to the [stability guide](https://github.com/Byron/gitoxide/blob/main/STABILITY.md)
 
+## Configuration and overrides
+
+As a general rule, respect and implement all applicable [git-config](https://git-scm.com/docs/git-config) by default, but allow the
+caller to set overrides. How overrides work depends on the goals of the particular API so it can be done on the main call path,
+forcing a choice, or more typically, as a side-lane where overrides can be done on demand.
+
+Note that it should be possible to obtain the current configuration for modification by the user for selective overrides, either
+by calling methods or by obtaining a data structure that can be set as a whole using a `get -> modify -> set` cycle.
+
+Note that without any of that, one should document that with `config_snapshot_mut()` any of the relevant configuration can be
+changed in memory before invoking a method in order to affect it.
+
+Parameters which are not available in git or specific to `gitoxide` or the needs of the caller can be passed as parameters or via
+`Options` or `Context` structures as needed.
+
 ## General
 
 * **async**

README.md

@@ -27,6 +27,9 @@ Please see _'Development Status'_ for a listing of all crates and their capabili
           * Based on the [git-hours] algorithm.
           * See the [discussion][git-hours-discussion] for some performance data.
   * **the `gix` program** _(plumbing)_ - lower level commands for use in automation
+    * **progress** - provide an overview of what works and what doesn't from the perspective of the git configuration.
+                     This is likely to change a lot over time depending on actual needs, but maybe useful for you to see
+                     if particular git-configuration is picked up and where it deviates.
     * **config** - list the complete git configuration in human-readable form and optionally filter sections by name.
     * **exclude**
         * [x] **query** - check if path specs are excluded via gits exclusion rules like `.gitignore`.

etc/check-package-size.sh

@@ -57,6 +57,6 @@ echo "in root: gitoxide CLI"
 (enter git-odb && indent cargo diet -n --package-size-limit 120KB)
 (enter git-protocol && indent cargo diet -n --package-size-limit 50KB)
 (enter git-packetline && indent cargo diet -n --package-size-limit 35KB)
-(enter git-repository && indent cargo diet -n --package-size-limit 175KB)
+(enter git-repository && indent cargo diet -n --package-size-limit 185KB)
 (enter git-transport && indent cargo diet -n --package-size-limit 55KB)
 (enter gitoxide-core && indent cargo diet -n --package-size-limit 90KB)

git-config/tests/file/init/from_paths/includes/conditional/onbranch.rs

@@ -285,6 +285,7 @@ value = branch-override-by-include
                 deref: false,
             }),
             git_repository::lock::acquire::Fail::Immediately,
+            git_repository::lock::acquire::Fail::Immediately,
         )?
         .commit(repo.committer_or_default())?;
 

git-pack/src/bundle/write/error.rs

@@ -2,7 +2,9 @@ use std::io;
 
 use git_tempfile::handle::Writable;
 
+/// The error returned by [`Bundle::write_to_directory()`][crate::Bundle::write_to_directory()]
 #[derive(thiserror::Error, Debug)]
+#[allow(missing_docs)]
 pub enum Error {
     #[error("An IO error occurred when reading the pack or creating a temporary file")]
     Io(#[from] io::Error),

git-pack/src/bundle/write/mod.rs

@@ -10,7 +10,7 @@ use git_tempfile::{handle::Writable, AutoRemove, ContainingDirectory};
 use crate::data;
 
 mod error;
-use error::Error;
+pub use error::Error;
 
 mod types;
 use types::{LockWriter, PassThrough};
@@ -228,7 +228,7 @@ impl crate::Bundle {
         Options {
             thread_limit,
             iteration_mode: _,
-            index_kind,
+            index_version: index_kind,
             object_hash,
         }: Options,
         data_file: Arc<parking_lot::Mutex<git_tempfile::Handle<Writable>>>,

git-pack/src/bundle/write/types.rs

@@ -11,7 +11,7 @@ pub struct Options {
     /// Determine how much processing to spend on protecting against corruption or recovering from errors.
     pub iteration_mode: crate::data::input::Mode,
     /// The version of pack index to write, should be [`crate::index::Version::default()`]
-    pub index_kind: crate::index::Version,
+    pub index_version: crate::index::Version,
     /// The kind of hash to use when writing the bundle.
     pub object_hash: git_hash::Kind,
 }
@@ -22,7 +22,7 @@ impl Default for Options {
         Options {
             thread_limit: None,
             iteration_mode: crate::data::input::Mode::Verify,
-            index_kind: Default::default(),
+            index_version: Default::default(),
             object_hash: Default::default(),
         }
     }

git-pack/src/index/write/error.rs

@@ -14,7 +14,7 @@ pub enum Error {
     IteratorInvariantNoRefDelta,
     #[error("The iterator failed to set a trailing hash over all prior pack entries in the last provided entry")]
     IteratorInvariantTrailer,
-    #[error("Did not encounter a single base")]
+    #[error("Did not encounter a single base - refusing to write empty pack.")]
     IteratorInvariantBasesPresent,
     #[error("Only u32::MAX objects can be stored in a pack, found {0}")]
     IteratorInvariantTooManyObjects(usize),

git-pack/src/index/write/mod.rs

@@ -18,7 +18,7 @@ pub(crate) struct TreeEntry {
 #[cfg_attr(feature = "serde1", derive(serde::Serialize, serde::Deserialize))]
 pub struct Outcome {
     /// The version of the verified index
-    pub index_kind: crate::index::Version,
+    pub index_version: crate::index::Version,
     /// The verified checksum of the verified index
     pub index_hash: git_hash::ObjectId,
 
@@ -171,7 +171,7 @@ impl crate::index::File {
                     modify_base(data, entry, bytes, kind.hash());
                     Ok::<_, Error>(())
                 },
-                crate::cache::delta::traverse::Options {
+                traverse::Options {
                     object_progress: root_progress.add_child("Resolving"),
                     size_progress: root_progress.add_child("Decoding"),
                     thread_limit,
@@ -207,20 +207,15 @@ impl crate::index::File {
             progress::MessageLevel::Success,
         );
         Ok(Outcome {
-            index_kind: kind,
+            index_version: kind,
             index_hash,
             data_hash: pack_hash,
             num_objects,
         })
     }
 }
 
-fn modify_base(
-    entry: &mut crate::index::write::TreeEntry,
-    pack_entry: &crate::data::Entry,
-    decompressed: &[u8],
-    hash: git_hash::Kind,
-) {
+fn modify_base(entry: &mut TreeEntry, pack_entry: &crate::data::Entry, decompressed: &[u8], hash: git_hash::Kind) {
     fn compute_hash(kind: git_object::Kind, bytes: &[u8], object_hash: git_hash::Kind) -> git_hash::ObjectId {
         let mut hasher = git_features::hash::hasher(object_hash);
         hasher.update(&git_object::encode::loose_header(kind, bytes.len()));

git-pack/tests/pack/bundle.rs

@@ -90,7 +90,7 @@ mod write_to_directory {
     fn expected_outcome() -> Result<pack::bundle::write::Outcome, Box<dyn std::error::Error>> {
         Ok(pack::bundle::write::Outcome {
             index: pack::index::write::Outcome {
-                index_kind: pack::index::Version::V2,
+                index_version: pack::index::Version::V2,
                 index_hash: git_hash::ObjectId::from_hex(b"544a7204a55f6e9cacccf8f6e191ea8f83575de3")?,
                 data_hash: git_hash::ObjectId::from_hex(b"0f3ea84cd1bba10c2a03d736a460635082833e59")?,
                 num_objects: 42,
@@ -156,7 +156,7 @@ mod write_to_directory {
             pack::bundle::write::Options {
                 thread_limit: None,
                 iteration_mode: pack::data::input::Mode::Verify,
-                index_kind: pack::index::Version::V2,
+                index_version: pack::index::Version::V2,
                 object_hash: git_hash::Kind::Sha1,
             },
         )

git-pack/tests/pack/data/output/count_and_entries.rs

@@ -321,6 +321,21 @@ fn traversals() -> crate::Result {
     Ok(())
 }
 
+#[test]
+fn empty_pack_is_not_allowed() {
+    assert_eq!(
+        write_and_verify(
+            db(DbKind::DeterministicGeneratedContent).unwrap(),
+            vec![],
+            hex_to_id("029d08823bd8a8eab510ad6ac75c823cfd3ed31e"),
+            None,
+        )
+        .unwrap_err()
+        .to_string(),
+        "Did not encounter a single base - refusing to write empty pack."
+    );
+}
+
 fn write_and_verify(
     db: git_odb::HandleArc,
     entries: Vec<output::Entry>,

git-pack/tests/pack/index.rs

@@ -205,7 +205,7 @@ mod version {
                     outcome.num_objects, num_objects,
                     "it wrote the entire iterator worth of entries"
                 );
-                assert_eq!(outcome.index_kind, desired_kind);
+                assert_eq!(outcome.index_version, desired_kind);
                 assert_eq!(
                     outcome.index_hash,
                     git_hash::ObjectId::from(&expected[end_of_pack_hash..end_of_index_hash])

git-protocol/src/fetch/arguments/blocking_io.rs

@@ -5,7 +5,8 @@ use git_transport::{client, client::TransportV2Ext};
 use crate::fetch::{Arguments, Command};
 
 impl Arguments {
-    pub(crate) fn send<'a, T: client::Transport + 'a>(
+    /// Send fetch arguments to the server, and indicate this is the end of negotiations only if `add_done_argument` is present.
+    pub fn send<'a, T: client::Transport + 'a>(
         &mut self,
         transport: &'a mut T,
         add_done_argument: bool,

git-protocol/src/fetch/arguments/mod.rs

@@ -3,6 +3,7 @@ use std::fmt;
 use bstr::{BStr, BString, ByteVec};
 
 /// The arguments passed to a server command.
+#[derive(Debug)]
 pub struct Arguments {
     /// The active features/capabilities of the fetch invocation
     #[cfg(any(feature = "async-client", feature = "blocking-client"))]
@@ -24,6 +25,13 @@ pub struct Arguments {
 }
 
 impl Arguments {
+    /// Return true if there is no argument at all.
+    ///
+    /// This can happen if callers assure that they won't add 'wants' if their 'have' is the same, i.e. if the remote has nothing
+    /// new for them.
+    pub fn is_empty(&self) -> bool {
+        self.args.is_empty()
+    }
     /// Return true if ref filters is supported.
     pub fn can_use_filter(&self) -> bool {
         self.filter
@@ -125,8 +133,10 @@ impl Arguments {
     fn prefixed(&mut self, prefix: &str, value: impl fmt::Display) {
         self.args.push(format!("{}{}", prefix, value).into());
     }
+    /// Create a new instance to help setting up arguments to send to the server as part of a `fetch` operation
+    /// for which `features` are the available and configured features to use.
     #[cfg(any(feature = "async-client", feature = "blocking-client"))]
-    pub(crate) fn new(version: git_transport::Protocol, features: Vec<crate::fetch::command::Feature>) -> Self {
+    pub fn new(version: git_transport::Protocol, features: Vec<crate::fetch::command::Feature>) -> Self {
         use crate::fetch::Command;
         let has = |name: &str| features.iter().any(|f| f.0 == name);
         let filter = has("filter");

git-protocol/src/fetch/command.rs

@@ -50,9 +50,12 @@ mod with_io {
                     "filter ", // filter-spec
                     // ref-in-want feature
                     "want-ref ", // ref path
+                    // sideband-all feature
                     "sideband-all",
                     // packfile-uris feature
                     "packfile-uris ", // protocols
+                    // wait-for-done feature
+                    "wait-for-done",
                 ],
             }
         }
@@ -79,9 +82,14 @@ mod with_io {
                         "no-done",
                         "filter",
                     ],
-                    git_transport::Protocol::V2 => {
-                        &["shallow", "filter", "ref-in-want", "sideband-all", "packfile-uris"]
-                    }
+                    git_transport::Protocol::V2 => &[
+                        "shallow",
+                        "filter",
+                        "ref-in-want",
+                        "sideband-all",
+                        "packfile-uris",
+                        "wait-for-done",
+                    ],
                 },
             }
         }
@@ -104,7 +112,9 @@ mod with_io {
             }
         }
 
-        pub(crate) fn default_features(
+        /// Turns on all modern features for V1 and all supported features for V2, returning them as a vector of features.
+        /// Note that this is the basis for any fetch operation as these features fulfil basic requirements and reasonably up-to-date servers.
+        pub fn default_features(
             &self,
             version: git_transport::Protocol,
             server_capabilities: &Capabilities,

git-protocol/src/fetch/handshake.rs

@@ -3,7 +3,7 @@ use git_transport::client::Capabilities;
 use crate::fetch::Ref;
 
 /// The result of the [`handshake()`][super::handshake()] function.
-#[derive(Debug, Clone)]
+#[derive(Default, Debug, Clone)]
 #[cfg_attr(feature = "serde1", derive(serde::Serialize, serde::Deserialize))]
 pub struct Outcome {
     /// The protocol version the server responded with. It might have downgraded the desired version.

git-protocol/src/fetch/refs/function.rs

@@ -10,13 +10,13 @@ use super::Error;
 use crate::fetch::{indicate_end_of_interaction, refs::from_v2_refs, Command, LsRefsAction, Ref};
 
 /// Invoke an ls-refs command on `transport`  (assuming `protocol_version` 2 or panic), which requires a prior handshake that yielded
-/// server `capabilities`. `prepare_ls_refs(arguments, features)` can be used to alter the _ls-refs_. `progress` is used to provide feedback.
+/// server `capabilities`. `prepare_ls_refs(capabilities, arguments, features)` can be used to alter the _ls-refs_. `progress` is used to provide feedback.
 #[maybe_async]
 pub async fn refs(
     mut transport: impl Transport,
     protocol_version: Protocol,
     capabilities: &Capabilities,
-    mut prepare_ls_refs: impl FnMut(
+    prepare_ls_refs: impl FnOnce(
         &Capabilities,
         &mut Vec<BString>,
         &mut Vec<(&str, Option<&str>)>,