Add :at_commit=sha[...] filter

Change: start-filter

Author: christian-schilling

docs/src/reference/filters.md

@@ -91,6 +91,13 @@ These filter do not modify git trees, but instead only operate on the commit gra
 Produce a filtered history that does not contain any merge commits. This is done by
 simply dropping all parents except the first on every commit.
 
+### Filter specific part of the history **:at_commit=<sha>[:filter]**
+Produce a history where the commit specified by `<sha>` is replaced by the result of applying
+`:filter` to it.
+This means also all parents of this specific commit appear filtered with `:filter` and all
+descendent commits will be left unchanged. However all commits hashes will still be different
+due to the filtered parents.
+
 Filter order matters
 --------------------
 

src/filter/grammar.pest

@@ -18,6 +18,7 @@ char = {
 
 filter_spec = { (
     filter_group
+  | filter_group_arg
   | filter_presub
   | filter_subdir
   | filter_nop
@@ -26,6 +27,7 @@ filter_spec = { (
 )+ }
 
 filter_group = { CMD_START ~ cmd? ~ GROUP_START ~ compose ~ GROUP_END }
+filter_group_arg = { CMD_START ~ cmd ~ "=" ~ argument ~ GROUP_START ~ compose ~ GROUP_END }
 filter_subdir = { CMD_START ~ "/" ~ argument }
 filter_nop = { CMD_START ~ "/" }
 filter_presub = { CMD_START ~ ":" ~ argument }

src/filter/mod.rs

@@ -98,6 +98,7 @@ enum Op {
     Fold,
     Paths,
     Squash(Option<std::collections::HashMap<git2::Oid, (String, String, String)>>),
+    AtCommit(git2::Oid, Filter),
     Linear,
 
     RegexReplace(regex::Regex, String),
@@ -194,6 +195,7 @@ fn nesting2(op: &Op) -> usize {
         Op::Workspace(_) => usize::MAX,
         Op::Chain(a, b) => 1 + nesting(*a).max(nesting(*b)),
         Op::Subtract(a, b) => 1 + nesting(*a).max(nesting(*b)),
+        Op::AtCommit(_, filter) => 1 + nesting(*filter),
         _ => 0,
     }
 }
@@ -223,6 +225,9 @@ fn spec2(op: &Op) -> String {
         Op::Exclude(b) => {
             format!(":exclude[{}]", spec(*b))
         }
+        Op::AtCommit(id, b) => {
+            format!(":at_commit={}[{}]", id, spec(*b))
+        }
         Op::Workspace(path) => {
             format!(":workspace={}", parse::quote(&path.to_string_lossy()))
         }
@@ -384,6 +389,18 @@ fn apply_to_commit2(
     rs_tracing::trace_scoped!("apply_to_commit", "spec": spec(filter), "commit": commit.id().to_string());
 
     let filtered_tree = match &to_op(filter) {
+        Op::AtCommit(id, startfilter) => {
+            if *id == commit.id() {
+                if let Some(start) = apply_to_commit2(&to_op(*startfilter), &commit, transaction)? {
+                    transaction.insert(filter, commit.id(), start, true);
+                    return Ok(Some(start));
+                } else {
+                    return Ok(None);
+                }
+            } else {
+                commit.tree()?
+            }
+        }
         Op::Squash(Some(ids)) => {
             if let Some(_) = ids.get(&commit.id()) {
                 commit.tree()?
@@ -619,6 +636,7 @@ fn apply2<'a>(
         Op::Squash(None) => Ok(tree),
         Op::Squash(Some(_)) => Err(josh_error("not applicable to tree")),
         Op::Linear => Ok(tree),
+        Op::AtCommit(_, _) => Err(josh_error("not applicable to tree")),
 
         Op::RegexReplace(regex, replacement) => {
             tree::regex_replace(tree.id(), &regex, &replacement, transaction)
@@ -712,7 +730,11 @@ pub fn unapply<'a>(
     parent_tree: git2::Tree<'a>,
 ) -> JoshResult<git2::Tree<'a>> {
     if let Ok(inverted) = invert(filter) {
-        let matching = apply(transaction, chain(filter, inverted), parent_tree.clone())?;
+        let matching = apply(
+            transaction,
+            chain(invert(inverted)?, inverted),
+            parent_tree.clone(),
+        )?;
         let stripped = tree::subtract(transaction, parent_tree.id(), matching.id())?;
         let new_tree = apply(transaction, inverted, tree)?;
 
@@ -733,7 +755,8 @@ pub fn unapply<'a>(
     }
 
     if let Op::Chain(a, b) = to_op(filter) {
-        let p = apply(transaction, a, parent_tree.clone())?;
+        let i = if let Ok(i) = invert(a) { invert(i)? } else { a };
+        let p = apply(transaction, i, parent_tree.clone())?;
         return unapply(
             transaction,
             a,

src/filter/opt.rs

@@ -328,6 +328,7 @@ fn step(filter: Filter) -> Filter {
                 Op::Prefix(path)
             }
         }
+        Op::AtCommit(id, filter) => Op::AtCommit(id, step(filter)),
         Op::Compose(filters) if filters.is_empty() => Op::Empty,
         Op::Compose(filters) if filters.len() == 1 => to_op(filters[0]),
         Op::Compose(mut filters) => {
@@ -419,6 +420,7 @@ pub fn invert(filter: Filter) -> JoshResult<Filter> {
         Op::File(path) => Some(Op::File(path)),
         Op::Prefix(path) => Some(Op::Subdir(path)),
         Op::Glob(pattern) => Some(Op::Glob(pattern)),
+        Op::AtCommit(_, _) => Some(Op::Nop),
         _ => None,
     };
 

src/filter/parse.rs

@@ -103,6 +103,23 @@ fn parse_item(pair: pest::iterators::Pair<Rule>) -> JoshResult<Op> {
                 _ => Err(josh_error("parse_item: no match {:?}")),
             }
         }
+        Rule::filter_group_arg => {
+            let v: Vec<_> = pair.into_inner().map(|x| unquote(x.as_str())).collect();
+
+            match v.as_slice() {
+                [cmd, arg, args] => {
+                    let g = parse_group(args)?;
+                    match *cmd {
+                        "at_commit" => Ok(Op::AtCommit(
+                            git2::Oid::from_str(arg)?,
+                            to_filter(Op::Compose(g)),
+                        )),
+                        _ => Err(josh_error("parse_item: no match")),
+                    }
+                }
+                _ => Err(josh_error("parse_item: no match {:?}")),
+            }
+        }
         _ => Err(josh_error("parse_item: no match")),
     }
 }

tests/filter/start.t

@@ -0,0 +1,52 @@
+  $ export RUST_BACKTRACE=1
+  $ git init -q 1> /dev/null
+
+  $ echo contents1 > file1
+  $ git add .
+  $ git commit -m "add file1" 1> /dev/null
+
+  $ git log --graph --pretty=%s
+  * add file1
+
+  $ git checkout -b branch2
+  Switched to a new branch 'branch2'
+
+  $ echo contents2 > file1
+  $ git add .
+  $ git commit -m "mod file1" 1> /dev/null
+
+  $ echo contents3 > file3
+  $ git add .
+  $ git commit -m "mod file3" 1> /dev/null
+
+  $ git checkout master
+  Switched to branch 'master'
+
+  $ echo contents3 > file2
+  $ git add .
+  $ git commit -m "add file2" 1> /dev/null
+
+  $ git merge -q branch2 --no-ff
+
+  $ git log --graph --pretty=%H
+  *   1d69b7d2651f744be3416f2ad526aeccefb99310
+  |\  
+  | * 86871b8775ad3baca86484337d1072aa1d386f7e
+  | * 975d4c4975912729482cc864d321c5196a969271
+  * | e707f76bb6a1390f28b2162da5b5eb6933009070
+  |/  
+  * 0b4cf6c9efbbda1eada39fa9c1d21d2525b027bb
+
+  $ josh-filter -s :at_commit=975d4c4975912729482cc864d321c5196a969271[:prefix=x/y] --update refs/heads/filtered
+  [2] :prefix=x
+  [2] :prefix=y
+  [5] :at_commit=975d4c4975912729482cc864d321c5196a969271[:prefix=x/y]
+
+  $ git log --graph --decorate --pretty=%H refs/heads/filtered
+  *   8b4097f3318cdf47e46266fc7fef5331bf189b6c
+  |\  
+  | * ee931ac07e4a953d1d2e0f65968946f5c09b0f4c
+  | * cc0382917c6488d69dca4d6a147d55251b06ac08
+  | * 9f0db868b59a422c114df33bc6a8b2950f80490b
+  * e707f76bb6a1390f28b2162da5b5eb6933009070
+  * 0b4cf6c9efbbda1eada39fa9c1d21d2525b027bb

tests/filter/subtree_prefix.t

@@ -0,0 +1,108 @@
+  $ git init -q 1>/dev/null
+
+Initial commit of main branch
+  $ echo contents1 > file1
+  $ git add .
+  $ git commit -m "add file1" 1>/dev/null
+
+Initial commit of subtree branch
+  $ git checkout --orphan subtree
+  Switched to a new branch 'subtree'
+  $ rm file*
+  $ echo contents2 > file2
+  $ git add .
+  $ git commit -m "add file2 (in subtree)" 1>/dev/null
+  $ export SUBTREE_TIP=$(git rev-parse HEAD)
+
+Articially create a subtree merge
+(merge commit has subtree files in subfolder but has subtree commit as a parent)
+  $ git checkout master
+  Switched to branch 'master'
+  $ git merge subtree --allow-unrelated-histories 1>/dev/null
+  $ mkdir subtree
+  $ git mv file2 subtree/
+  $ git add subtree
+  $ git commit -a --amend -m "subtree merge" 1>/dev/null
+  $ tree
+  .
+  |-- file1
+  `-- subtree
+      `-- file2
+  
+  1 directory, 2 files
+  $ git log --graph --pretty=%s
+  *   subtree merge
+  |\  
+  | * add file2 (in subtree)
+  * add file1
+
+Change subtree file
+  $ echo more contents >> subtree/file2
+  $ git commit -a -m "subtree edit from main repo" 1>/dev/null
+
+Rewrite the subtree part of the history
+  $ josh-filter -s :at_commit=$SUBTREE_TIP[:prefix=subtree] refs/heads/master --update refs/heads/filtered
+  [1] :prefix=subtree
+  [4] :at_commit=c036f944faafb865e0585e4fa5e005afa0aeea3f[:prefix=subtree]
+
+  $ git log --graph --pretty=%s refs/heads/filtered
+  * subtree edit from main repo
+  *   subtree merge
+  |\  
+  | * add file2 (in subtree)
+  * add file1
+
+Compare input and result. ^^2 is the 2nd parent of the first parent, i.e., the 'in subtree' commit.
+  $ git ls-tree --name-only -r refs/heads/filtered
+  file1
+  subtree/file2
+  $ git diff refs/heads/master refs/heads/filtered
+  $ git ls-tree --name-only -r refs/heads/filtered^^2
+  subtree/file2
+  $ git diff refs/heads/master^^2 refs/heads/filtered^^2
+  diff --git a/file2 b/subtree/file2
+  similarity index 100%
+  rename from file2
+  rename to subtree/file2
+
+Extract the subtree history
+  $ josh-filter -s :at_commit=$SUBTREE_TIP[:prefix=subtree]:/subtree refs/heads/master --update refs/heads/subtree
+  [1] :prefix=subtree
+  [4] :/subtree
+  [4] :at_commit=c036f944faafb865e0585e4fa5e005afa0aeea3f[:prefix=subtree]
+  $ git checkout subtree
+  Switched to branch 'subtree'
+  $ cat file2
+  contents2
+  more contents
+
+Work in the subtree, and sync that back.
+  $ echo even more contents >> file2
+  $ git commit -am "add even more content" 1>/dev/null
+  $ josh-filter -s :at_commit=$SUBTREE_TIP[:prefix=subtree]:/subtree refs/heads/master --update refs/heads/subtree --reverse
+  [1] :prefix=subtree
+  [4] :/subtree
+  [4] :at_commit=c036f944faafb865e0585e4fa5e005afa0aeea3f[:prefix=subtree]
+  $ git log --graph --pretty=%s  refs/heads/master
+  * add even more content
+  * subtree edit from main repo
+  *   subtree merge
+  |\  
+  | * add file2 (in subtree)
+  * add file1
+  $ git ls-tree --name-only -r refs/heads/master
+  file1
+  subtree/file2
+  $ git checkout master
+  Switched to branch 'master'
+  $ cat subtree/file2
+  contents2
+  more contents
+  even more contents
+
+And then re-extract, which should re-construct the same subtree.
+  $ josh-filter -s :at_commit=$SUBTREE_TIP[:prefix=subtree]:/subtree refs/heads/master --update refs/heads/subtree2
+  [1] :prefix=subtree
+  [5] :/subtree
+  [5] :at_commit=c036f944faafb865e0585e4fa5e005afa0aeea3f[:prefix=subtree]
+  $ test $(git rev-parse subtree) = $(git rev-parse subtree2)

tests/proxy/workspace_errors.t

@@ -104,7 +104,7 @@ Error in filter
   remote: 1 | a/b = :b/sub2        
   remote:   |         ^---        
   remote:   |        
-  remote:   = expected EOI, filter_group, filter_subdir, filter_nop, filter_presub, filter, or filter_noarg        
+  remote:   = expected EOI, filter_group, filter_group_arg, filter_subdir, filter_nop, filter_presub, filter, or filter_noarg        
   remote: 
   remote: a/b = :b/sub2        
   remote: c = :/sub1