index reading

Cargo.toml

@@ -115,6 +115,7 @@ members = [
     "git-diff",
     "git-traverse",
     "git-index",
+    "git-worktree",
     "git-packetline",
     "git-transport",
     "git-protocol",

Makefile

@@ -82,6 +82,7 @@ check: ## Build all code in suitable configurations
 				&& cargo check
 	cd git-object && cargo check --all-features \
                   && cargo check --features verbose-object-parsing-errors
+	cd git-index && cargo check --features serde1
 	cd git-actor && cargo check --features serde1
 	cd git-pack && cargo check --features serde1 \
 			   && cargo check --features pack-cache-lru-static \
@@ -139,6 +140,8 @@ unit-tests: ## run all unit tests
 	cd git-object && cargo test && cargo test --features verbose-object-parsing-errors
 	cd git-pack && cargo test --features internal-testing-to-avoid-being-run-by-cargo-test-all \
 				&& cargo test --features "internal-testing-git-features-parallel"
+	cd git-index && cargo test --features internal-testing-to-avoid-being-run-by-cargo-test-all \
+				&& cargo test --features "internal-testing-git-features-parallel"
 	cd git-packetline && cargo test \
 					  && cargo test --features blocking-io,maybe-async/is_sync --test blocking-packetline \
 					  && cargo test --features "async-io" --test async-packetline

README.md

@@ -83,9 +83,9 @@ Follow linked crate name for detailed status. Please note that all crates follow
   * [git-repository](https://github.com/Byron/gitoxide/blob/main/crate-status.md#git-repository)
   * `gitoxide-core`
 * **very early**    
-* **idea**
   * [git-index](https://github.com/Byron/gitoxide/blob/main/crate-status.md#git-index)
-  * git-status
+* **idea**
+  * [git-worktree](https://github.com/Byron/gitoxide/blob/main/crate-status.md#git-worktree)
   * [git-tui](https://github.com/Byron/gitoxide/blob/main/crate-status.md#git-tui)
   * [git-bundle](https://github.com/Byron/gitoxide/blob/main/crate-status.md#git-bundle)
 
@@ -240,14 +240,9 @@ Provide a CLI to for the most basic user journey:
 
 * [ ] `gix tool open-remote` open the URL of the remote, possibly after applying known transformations to go from `ssh` to `https`.
 * [ ] Open up SQL for git using [sqlite virtual tables](https://github.com/rusqlite/rusqlite/blob/master/tests/vtab.rs). Check out gitqlite
-  as well. What would an MVP look like? Maybe even something that could ship with gitoxide.
+  as well. What would an MVP look like? Maybe even something that could ship with gitoxide. See [this go implementation as example](https://github.com/filhodanuvem/gitql).
 * [ ] A truly awesome history rewriter which makes it easy to understand what happened while avoiding all pitfalls. Think BFG, but more awesome, if that's possible.
 * [ ] `git-tui` should learn a lot from [fossil-scm] regarding the presentation of data. Maybe [this](https://github.com/Lutetium-Vanadium/requestty/) can be used for prompts. Probably [magit] has a lot to offer, too.
-* [ ] Can markdown be used as database so issue-trackers along with meta-data could just be markdown files which are mostly human-editable? Could user interfaces
-  be meta-data aware and just hide the meta-data chunks which are now editable in the GUI itself? Doing this would make conflicts easier to resolve than an `sqlite`
-  database.
-    * ~~A git-backend for `sqlite` which should allow embedding sqlite databases into git repositories, which in turn can be used for bug-trackers, wikis or other
-      features, making for a fully distributed github like experience, maybe.~~
 
 ### Ideas for Spin-Offs
 
@@ -259,6 +254,18 @@ Provide a CLI to for the most basic user journey:
 * [ ] A [syncthing] like client/server application. This is to demonstrate how lower-level crates can be combined into custom applications that use
   only part of git's technology to achieve their very own thing. Watch out for big file support, multi-device cross-syncing, the possibility for
   untrusted destinations using full-encryption, case-insensitive and sensitive filesystems, and extended file attributes as well as ignore files.
+* An event-based database that uses commit messages to store deltas, while occasionally aggregating the actual state in a tree. Of course it's distributed by nature, allowing
+  people to work offline.
+    - It's abstracted to completely hide the actual data model behind it, allowing for all kinds of things to be implemented on top.
+    - Commits probably need a nanosecond component for the timestamp, which can be added via custom header field.
+    - having recording all changes allows for perfect merging, both on the client or on the server, while keeping a natural audit log which makes it useful for mission critical
+      databases in business.
+    * **Applications**
+      - Can markdown be used as database so issue-trackers along with meta-data could just be markdown files which are mostly human-editable? Could user interfaces
+        be meta-data aware and just hide the meta-data chunks which are now editable in the GUI itself? Doing this would make conflicts easier to resolve than an `sqlite`
+        database.
+      - A time tracker - simple data, very likely naturally conflict free, and interesting to see it in terms of teams or companies using it with maybe GitHub as Backing for authentication.
+        - How about supporting multiple different trackers, as in different remotes?
 
 [syncthing]: https://github.com/syncthing/syncthing
 [fossil-scm]: https://www.fossil-scm.org

crate-status.md

@@ -206,10 +206,35 @@ Check out the [performance discussion][git-traverse-performance] as well.
 * [x] API documentation
     * [ ] Some examples
 
+### git-worktree
+* handle the working tree/checkout
+* manage multiple worktrees
+* deal with exclude specifications, like .gitignore and other exclude files.
+
 ### git-index
-* read and write a git-index file
-    * non-sparse
-    * sparse (search for [`sparse index` here](https://github.blog/2021-08-16-highlights-from-git-2-33/))
+* read 
+  * [ ] V2
+  * [ ] V3
+  * [ ] V4
+  * optional threading
+    * [ ] concurrent loading of index extensions
+    * [ ] threaded cache entry reading
+* `stat` update
+    * [ ] optional threaded `stat` based on thread_cost (aka preload)
+* [ ] handling of `.gitignore` and system file exclude configuration
+* [ ] handle potential races
+* extensions
+    * [ ] TREE for speeding up tree generation
+    * [ ] REUC resolving undo
+    * [ ] UNTR untracked cache
+    * [ ] FSMN file system monitor cache V1 and V2
+    * [ ] EOIE end of index entry
+    * [ ] IEOT index entry offset table
+    * [ ] link base indices to take information from, split index
+    * [ ] sdir sparse directory entries
+* additinoal support
+    * [ ] non-sparse
+    * [ ] sparse (search for [`sparse index` here](https://github.blog/2021-08-16-highlights-from-git-2-33/))
 * add and remove entries
 * [x] API documentation
     * [ ] Some examples

etc/check-package-size.sh

@@ -18,12 +18,13 @@ echo "in root: gitoxide CLI"
 #indent cargo diet -n --package-size-limit 25KB - fails right now because of dotted profile.dev.package
 (enter cargo-smart-release && indent cargo diet -n --package-size-limit 85KB)
 (enter git-actor && indent cargo diet -n --package-size-limit 5KB)
+(enter git-index && indent cargo diet -n --package-size-limit 15KB)
 (enter git-tempfile && indent cargo diet -n --package-size-limit 25KB)
 (enter git-lock && indent cargo diet -n --package-size-limit 15KB)
 (enter git-config && indent cargo diet -n --package-size-limit 65KB)
 (enter git-hash && indent cargo diet -n --package-size-limit 10KB)
 (enter git-chunk && indent cargo diet -n --package-size-limit 10KB)
-(enter git-features && indent cargo diet -n --package-size-limit 35KB)
+(enter git-features && indent cargo diet -n --package-size-limit 40KB)
 (enter git-ref && indent cargo diet -n --package-size-limit 50KB)
 (enter git-diff && indent cargo diet -n --package-size-limit 10KB)
 (enter git-traverse && indent cargo diet -n --package-size-limit 10KB)

git-commitgraph/Cargo.toml

@@ -21,8 +21,7 @@ git-hash = { version ="^0.8.0", path = "../git-hash" }
 git-chunk = { version ="^0.2.0", path = "../git-chunk" }
 
 bstr = { version = "0.2.13", default-features = false, features = ["std"] }
-byteorder = "1.2.3"
-filebuffer = "0.4.0"
+memmap2 = "0.5.0"
 serde = { version = "1.0.114", optional = true, default-features = false, features = ["derive"] }
 thiserror = "1.0.26"
 

git-commitgraph/src/file/commit.rs

@@ -5,8 +5,6 @@ use std::{
     slice::Chunks,
 };
 
-use byteorder::{BigEndian, ByteOrder};
-
 use crate::{
     file::{self, File, EXTENDED_EDGES_MASK, LAST_EXTENDED_EDGE_MASK, NO_PARENT},
     graph,
@@ -38,17 +36,23 @@ pub struct Commit<'a> {
     root_tree_id: &'a git_hash::oid,
 }
 
+#[inline]
+fn read_u32(b: &[u8]) -> u32 {
+    u32::from_be_bytes(b.try_into().unwrap())
+}
+
 impl<'a> Commit<'a> {
     pub(crate) fn new(file: &'a File, pos: file::Position) -> Self {
         let bytes = file.commit_data_bytes(pos);
         Commit {
             file,
             pos,
             root_tree_id: git_hash::oid::from_bytes_unchecked(&bytes[..file.hash_len]),
-            parent1: ParentEdge::from_raw(BigEndian::read_u32(&bytes[file.hash_len..][..4])),
-            parent2: ParentEdge::from_raw(BigEndian::read_u32(&bytes[file.hash_len + 4..][..4])),
-            generation: BigEndian::read_u32(&bytes[file.hash_len + 8..][..4]) >> 2,
-            commit_timestamp: BigEndian::read_u64(&bytes[file.hash_len + 8..][..8]) & 0x0003_ffff_ffff,
+            parent1: ParentEdge::from_raw(read_u32(&bytes[file.hash_len..][..4])),
+            parent2: ParentEdge::from_raw(read_u32(&bytes[file.hash_len + 4..][..4])),
+            generation: read_u32(&bytes[file.hash_len + 8..][..4]) >> 2,
+            commit_timestamp: u64::from_be_bytes(bytes[file.hash_len + 8..][..8].try_into().unwrap())
+                & 0x0003_ffff_ffff,
         }
     }
 
@@ -173,7 +177,7 @@ impl<'a> Iterator for ParentIterator<'a> {
             },
             ParentIteratorState::Extra(mut chunks) => {
                 if let Some(chunk) = chunks.next() {
-                    let extra_edge = BigEndian::read_u32(chunk);
+                    let extra_edge = read_u32(chunk);
                     match ExtraEdge::from_raw(extra_edge) {
                         ExtraEdge::Internal(pos) => {
                             self.state = ParentIteratorState::Extra(chunks);

git-commitgraph/src/file/init.rs

@@ -4,8 +4,7 @@ use std::{
 };
 
 use bstr::ByteSlice;
-use byteorder::{BigEndian, ByteOrder};
-use filebuffer::FileBuffer;
+use memmap2::Mmap;
 
 use crate::file::{
     ChunkId, File, BASE_GRAPHS_LIST_CHUNK_ID, COMMIT_DATA_CHUNK_ID, COMMIT_DATA_ENTRY_SIZE_SANS_HASH,
@@ -66,10 +65,18 @@ impl TryFrom<&Path> for File {
     type Error = Error;
 
     fn try_from(path: &Path) -> Result<Self, Self::Error> {
-        let data = FileBuffer::open(path).map_err(|e| Error::Io {
-            err: e,
-            path: path.to_owned(),
-        })?;
+        let data = std::fs::File::open(path)
+            .and_then(|file| {
+                // SAFETY: we have to take the risk of somebody changing the file underneath. Git never writes into the same file.
+                #[allow(unsafe_code)]
+                unsafe {
+                    Mmap::map(&file)
+                }
+            })
+            .map_err(|e| Error::Io {
+                err: e,
+                path: path.to_owned(),
+            })?;
         let data_size = data.len();
         if data_size < MIN_FILE_SIZE {
             return Err(Error::Corrupt(
@@ -241,7 +248,7 @@ impl TryFrom<&Path> for File {
 fn read_fan(d: &[u8]) -> ([u32; FAN_LEN], usize) {
     let mut fan = [0; FAN_LEN];
     for (c, f) in d.chunks(4).zip(fan.iter_mut()) {
-        *f = BigEndian::read_u32(c);
+        *f = u32::from_be_bytes(c.try_into().unwrap());
     }
     (fan, FAN_LEN * 4)
 }

git-commitgraph/src/file/mod.rs

@@ -6,7 +6,7 @@ use std::{
     path::PathBuf,
 };
 
-use filebuffer::FileBuffer;
+use memmap2::Mmap;
 
 pub use self::{commit::Commit, init::Error};
 
@@ -42,7 +42,7 @@ pub struct File {
     base_graph_count: u8,
     base_graphs_list_offset: Option<usize>,
     commit_data_offset: usize,
-    data: FileBuffer,
+    data: Mmap,
     extra_edges_list_range: Option<Range<usize>>,
     fan: [u32; FAN_LEN],
     oid_lookup_offset: usize,

git-commitgraph/src/lib.rs

@@ -7,8 +7,7 @@
 //! As generating the full commit graph from scratch can take some time, git may write new commits
 //! to separate [files][file::File] instead of overwriting the original file.
 //! Eventually, git will merge these files together as the number of files grows.
-#![forbid(unsafe_code)]
-#![deny(rust_2018_idioms, missing_docs)]
+#![deny(unsafe_code, rust_2018_idioms, missing_docs)]
 
 pub mod file;
 pub mod graph;