From 5ac19164880e8d316223e3553e206344e6e508d0 Mon Sep 17 00:00:00 2001 From: shbhmexe Date: Tue, 23 Dec 2025 12:48:13 +0530 Subject: [PATCH] fix(runners): repair gitdm wrappers and runner script paths Several runner scripts referenced ~/dev/cncf/gitdm/cncfdm.py, but the engine lives under src/cncfdm.py. This breaks common workflows driven by rerun_data.sh (run_no_map/run_with_map, strict/no-map release runs, multirepo runs, and commits-in-range analysis) unless the user has a custom copy/symlink. Also fix gitdm.ps1 on Windows PowerShell: - remove bash heredoc usage - correctly invoke "py -2" with arguments - forward piped stdin to cncfdm.py when used in a pipeline Signed-off-by: shbhmexe --- gitdm.ps1 | 61 ++++++++++++++++++++++++-------------- src/all_no_map.sh | 3 +- src/all_with_map.sh | 3 +- src/anyrepo.sh | 4 ++- src/anyreporange.sh | 4 ++- src/commits_in_ranges.sh | 12 ++++++-- src/multirepo.sh | 8 +++-- src/rerun_data.sh | 2 +- src/run_all.sh | 3 +- src/run_for_rels_no_map.sh | 11 ++++--- src/run_for_rels_strict.sh | 11 ++++--- src/run_no_map.sh | 10 ++++--- src/run_with_map.sh | 9 ++++-- 13 files changed, 92 insertions(+), 49 deletions(-) diff --git a/gitdm.ps1 b/gitdm.ps1 index a35f2bb8..a2602b4b 100644 --- a/gitdm.ps1 +++ b/gitdm.ps1 @@ -5,37 +5,54 @@ $ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path $BaseDir = Join-Path $ScriptDir 'src' $Engine = Join-Path $BaseDir 'cncfdm.py' +function Test-Python2 { + param( + [Parameter(Mandatory=$true)] [string]$Exe, + [string[]]$ExeArgs = @() + ) + try { + & $Exe @ExeArgs -c "import sys; sys.exit(0 if sys.version_info[0] == 2 else 1)" *> $null + return ($LASTEXITCODE -eq 0) + } catch { + return $false + } +} + function Pick-Python { - if ($env:GITDM_PY) { return $env:GITDM_PY } - foreach ($cand in @('py -2','python2','pypy','pypy2')) { - try { - if ($cand -eq 'py -2') { - & py -2 - <<'PY' -import sys -print(2 if sys.version_info[0]==2 else 3) -PY - if ($LASTEXITCODE -eq 0) { return 'py -2' } - } else { - if (Get-Command ($cand.Split(' ')[0]) -ErrorAction SilentlyContinue) { return $cand } - } - } catch { } + # Allow override via environment variable. + if ($env:GITDM_PY) { + return [pscustomobject]@{ Exe = $env:GITDM_PY; Args = @() } + } + + # Prefer Python Launcher (if Python 2 is installed). + if (Get-Command py -ErrorAction SilentlyContinue) { + if (Test-Python2 -Exe 'py' -ExeArgs @('-2')) { + return [pscustomobject]@{ Exe = 'py'; Args = @('-2') } + } } - if (Get-Command python -ErrorAction SilentlyContinue) { - $ver = & python - <<'PY' -import sys -print(sys.version_info[0]) -PY - if ($ver -eq 2) { return 'python' } + + # Prefer explicit Python 2 binaries, then fall back to python only if it is v2. + foreach ($cand in @('python2', 'pypy2', 'pypy', 'python')) { + if (Get-Command $cand -ErrorAction SilentlyContinue) { + if (Test-Python2 -Exe $cand) { + return [pscustomobject]@{ Exe = $cand; Args = @() } + } + } } + return $null } $py = Pick-Python if (-not $py) { - Write-Error 'gitdm requires Python 2 or PyPy. Install python2/pypy or set GITDM_PY.' + Write-Error 'gitdm requires Python 2 or PyPy. Install python2/pypy2 or set GITDM_PY.' exit 1 } # Pass through stdin/stdout; add -b to point at src/ -& $py $Engine -b "$BaseDir/" @Args -exit $LASTEXITCODE \ No newline at end of file +if ($MyInvocation.ExpectingInput) { + $input | & $py.Exe @($py.Args) $Engine -b "$BaseDir/" @Args +} else { + & $py.Exe @($py.Args) $Engine -b "$BaseDir/" @Args +} +exit $LASTEXITCODE diff --git a/src/all_no_map.sh b/src/all_no_map.sh index 99d8e837..0f201b4b 100755 --- a/src/all_no_map.sh +++ b/src/all_no_map.sh @@ -1,2 +1,3 @@ +GITDM_HOME=${GITDM_HOME:-`cd "$(dirname "$0")/.." && pwd`} ./run_no_map.sh -mv run_no_map_* ~/dev/cncf/gitdm/kubernetes/all_time/ +mv run_no_map_* "$GITDM_HOME/kubernetes/all_time/" diff --git a/src/all_with_map.sh b/src/all_with_map.sh index 1a9a4850..7c771377 100755 --- a/src/all_with_map.sh +++ b/src/all_with_map.sh @@ -1,2 +1,3 @@ +GITDM_HOME=${GITDM_HOME:-`cd "$(dirname "$0")/.." && pwd`} ./run_with_map.sh -mv run_with_map_* ~/dev/cncf/gitdm/kubernetes/all_time/ +mv run_with_map_* "$GITDM_HOME/kubernetes/all_time/" diff --git a/src/anyrepo.sh b/src/anyrepo.sh index 737e876e..fbad260d 100755 --- a/src/anyrepo.sh +++ b/src/anyrepo.sh @@ -1,10 +1,12 @@ #!/bin/sh PWD=`pwd` FN=$PWD/repos/$2 +GITDM_HOME=${GITDM_HOME:-`cd "$(dirname "$0")/.." && pwd`} +CNCFDM="$GITDM_HOME/src/cncfdm.py" cd "$1" echo "Processing repo $1 $2" git config merge.renameLimit 100000 git config diff.renameLimit 100000 -git log --all --numstat -M | ~/dev/cncf/gitdm/cncfdm.py -r '^vendor/|/vendor/|^Godeps/' -R -n -b ~/dev/cncf/gitdm/ -t -z -d -D -U -u -h $FN.html -o $FN.txt -x $FN.csv > $FN.out +git log --all --numstat -M | "$CNCFDM" -r '^vendor/|/vendor/|^Godeps/' -R -n -b "$GITDM_HOME/src/" -t -z -d -D -U -u -h $FN.html -o $FN.txt -x $FN.csv > $FN.out git config --unset diff.renameLimit git config --unset merge.renameLimit diff --git a/src/anyreporange.sh b/src/anyreporange.sh index 53f44d97..5f1346a6 100755 --- a/src/anyreporange.sh +++ b/src/anyreporange.sh @@ -1,9 +1,11 @@ #!/bin/sh PWD=`pwd` FN=$PWD/repo_$2_$3 +GITDM_HOME=${GITDM_HOME:-`cd "$(dirname "$0")/.." && pwd`} +CNCFDM="$GITDM_HOME/src/cncfdm.py" cd "$1" git config merge.renameLimit 100000 git config diff.renameLimit 100000 -git log --all --numstat -M --since "$2" --until "$3" | ~/dev/cncf/gitdm/cncfdm.py -r '^vendor/|/vendor/|^Godeps/' -R -n -b ~/dev/cncf/gitdm/ -t -z -d -D -U -u -f "$2" -e "$3" -h $FN.html -o $FN.txt -x $FN.csv > $FN.out +git log --all --numstat -M --since "$2" --until "$3" | "$CNCFDM" -r '^vendor/|/vendor/|^Godeps/' -R -n -b "$GITDM_HOME/src/" -t -z -d -D -U -u -f "$2" -e "$3" -h $FN.html -o $FN.txt -x $FN.csv > $FN.out git config --unset diff.renameLimit git config --unset merge.renameLimit diff --git a/src/commits_in_ranges.sh b/src/commits_in_ranges.sh index e7659c7b..256c8a18 100755 --- a/src/commits_in_ranges.sh +++ b/src/commits_in_ranges.sh @@ -1,4 +1,8 @@ #!/bin/sh +if [ $# -lt 4 ]; then + echo "Usage: $0 prefix git.log dt_from dt_to" + exit 1 +fi echo "Args git.log dt_from dt_to" WD=`pwd` PREFIX=$1 @@ -6,6 +10,8 @@ FN=$2 F=$WD/other_repos/$1_range_unknown_$3_$4 F2=$WD/other_repos/$1_range_no_map_$3_$4 F3=$WD/other_repos/$1_range_with_map_$3_$4 -cat $FN | ~/dev/cncf/gitdm/cncfdm.py -f "$3" -e "$4" -r '^vendor/|/vendor/|^Godeps/' -R -n -b ~/dev/cncf/gitdm/ -t -z -d -D -U -u -h $F.html -o $F.txt -x $F.csv > $F.out -#cat $FN | ~/dev/cncf/gitdm/cncfdm.py -f "$3" -e "$4" -r '^vendor/|/vendor/|^Godeps/' -R -n -b ~/dev/cncf/gitdm/ -t -z -d -D -U -h $F2.html -o $F2.txt -x $F2.csv > $F2.out -#cat $FN | ~/dev/cncf/gitdm/cncfdm.py -f "$3" -e "$4" -r '^vendor/|/vendor/|^Godeps/' -R -n -b ~/dev/cncf/gitdm/ -t -z -d -D -U -m -h $F3.html -o $F3.txt -x $F3.csv > $F3.out +GITDM_HOME=${GITDM_HOME:-`cd "$(dirname "$0")/.." && pwd`} +CNCFDM="$GITDM_HOME/src/cncfdm.py" +cat $FN | "$CNCFDM" -f "$3" -e "$4" -r '^vendor/|/vendor/|^Godeps/' -R -n -b "$GITDM_HOME/src/" -t -z -d -D -U -u -h $F.html -o $F.txt -x $F.csv > $F.out +#cat $FN | "$CNCFDM" -f "$3" -e "$4" -r '^vendor/|/vendor/|^Godeps/' -R -n -b "$GITDM_HOME/src/" -t -z -d -D -U -h $F2.html -o $F2.txt -x $F2.csv > $F2.out +#cat $FN | "$CNCFDM" -f "$3" -e "$4" -r '^vendor/|/vendor/|^Godeps/' -R -n -b "$GITDM_HOME/src/" -t -z -d -D -U -m -h $F3.html -o $F3.txt -x $F3.csv > $F3.out diff --git a/src/multirepo.sh b/src/multirepo.sh index 524b359a..675a4eba 100755 --- a/src/multirepo.sh +++ b/src/multirepo.sh @@ -4,6 +4,8 @@ FN=$WD/git.log F=$WD/repos/combined F2=$WD/repos/combined_no_map F3=$WD/repos/combined_with_map +GITDM_HOME=${GITDM_HOME:-`cd "$(dirname "$0")/.." && pwd`} +CNCFDM="$GITDM_HOME/src/cncfdm.py" > $FN for var in "$@" do @@ -18,9 +20,9 @@ do done PWD=$WD cd $PWD -cat $FN | ~/dev/cncf/gitdm/cncfdm.py -r '^vendor/|/vendor/|^Godeps/' -R -n -b ~/dev/cncf/gitdm/ -t -z -d -D -U -u -h $F.html -o $F.txt -x $F.csv > $F.out -cat $FN | ~/dev/cncf/gitdm/cncfdm.py -r '^vendor/|/vendor/|^Godeps/' -R -n -b ~/dev/cncf/gitdm/ -t -z -d -D -U -h $F2.html -o $F2.txt -x $F2.csv > $F2.out -cat $FN | ~/dev/cncf/gitdm/cncfdm.py -r '^vendor/|/vendor/|^Godeps/' -R -n -b ~/dev/cncf/gitdm/ -t -z -d -D -U -m -h $F3.html -o $F3.txt -x $F3.csv > $F3.out +cat $FN | "$CNCFDM" -r '^vendor/|/vendor/|^Godeps/' -R -n -b "$GITDM_HOME/src/" -t -z -d -D -U -u -h $F.html -o $F.txt -x $F.csv > $F.out +cat $FN | "$CNCFDM" -r '^vendor/|/vendor/|^Godeps/' -R -n -b "$GITDM_HOME/src/" -t -z -d -D -U -h $F2.html -o $F2.txt -x $F2.csv > $F2.out +cat $FN | "$CNCFDM" -r '^vendor/|/vendor/|^Godeps/' -R -n -b "$GITDM_HOME/src/" -t -z -d -D -U -m -h $F3.html -o $F3.txt -x $F3.csv > $F3.out ./commits_in_default_ranges.sh all_kubernetes $FN #rm -f $FN #xz -9 $FN diff --git a/src/rerun_data.sh b/src/rerun_data.sh index aebf7973..08f7252b 100755 --- a/src/rerun_data.sh +++ b/src/rerun_data.sh @@ -4,7 +4,7 @@ echo "Update PULL_DATE after this, and update datasheet dates and report dates t echo "Also update last_processed.txt with last processed unknown/not found affliation from repos/combined.txt" rm -f ./other_repos/* echo "All with map to (Unknown)" -./all.sh +./run_all.sh echo "All without mapping" ./all_no_map.sh echo "All with map to Domain *" diff --git a/src/run_all.sh b/src/run_all.sh index 50c87acf..079aae05 100755 --- a/src/run_all.sh +++ b/src/run_all.sh @@ -1,2 +1,3 @@ +GITDM_HOME=${GITDM_HOME:-`cd "$(dirname "$0")/.." && pwd`} ./run.sh -mv first_run_* ~/dev/cncf/gitdm/kubernetes/all_time/ +mv first_run_* "$GITDM_HOME/kubernetes/all_time/" diff --git a/src/run_for_rels_no_map.sh b/src/run_for_rels_no_map.sh index 54335d72..312c6d56 100755 --- a/src/run_for_rels_no_map.sh +++ b/src/run_for_rels_no_map.sh @@ -1,18 +1,21 @@ #!/bin/sh if [ $# -lt 2 ]; then echo "$0 tag1 tag2" - echo "Use "git tag -l" to see available tags" + echo 'Use "git tag -l" to see available tags' exit 1 fi PWD=`pwd` FNP=$PWD/output_no_map_patch FNN=$PWD/output_no_map_numstat -cd ~/dev/go/src/k8s.io/kubernetes/ +GITDM_HOME=${GITDM_HOME:-`cd "$(dirname "$0")/.." && pwd`} +K8S_REPO=${K8S_REPO:-$HOME/dev/go/src/k8s.io/kubernetes/} +CNCFDM="$GITDM_HOME/src/cncfdm.py" +cd "$K8S_REPO" git config merge.renameLimit 100000 git config diff.renameLimit 100000 # -m --> map unknowns to 'DomainName *' , -u map unknowns to '(Unknown)' -git log --all -p -M $1..$2 | ~/dev/cncf/gitdm/cncfdm.py -r '^vendor/|/vendor/|^Godeps/' -R -b ~/dev/cncf/gitdm/ -t -z -d -D -U -h $FNP.html -o $FNP.txt -x $FNP.csv -git log --all --numstat -M $1..$2 | ~/dev/cncf/gitdm/cncfdm.py -r '^vendor/|/vendor/|^Godeps/' -R -n -b ~/dev/cncf/gitdm/ -t -z -d -D -U -h $FNN.html -o $FNN.txt -x $FNN.csv > $FNN.out +git log --all -p -M $1..$2 | "$CNCFDM" -r '^vendor/|/vendor/|^Godeps/' -R -b "$GITDM_HOME/src/" -t -z -d -D -U -h $FNP.html -o $FNP.txt -x $FNP.csv +git log --all --numstat -M $1..$2 | "$CNCFDM" -r '^vendor/|/vendor/|^Godeps/' -R -n -b "$GITDM_HOME/src/" -t -z -d -D -U -h $FNN.html -o $FNN.txt -x $FNN.csv > $FNN.out git config --unset diff.renameLimit git config --unset merge.renameLimit ls -l $FNP* $FNN* diff --git a/src/run_for_rels_strict.sh b/src/run_for_rels_strict.sh index 673d2278..454d3990 100755 --- a/src/run_for_rels_strict.sh +++ b/src/run_for_rels_strict.sh @@ -1,18 +1,21 @@ #!/bin/sh if [ $# -lt 2 ]; then echo "$0 tag1 tag2" - echo "Use "git tag -l" to see available tags" + echo 'Use "git tag -l" to see available tags' exit 1 fi PWD=`pwd` FNP=$PWD/output_strict_patch FNN=$PWD/output_strict_numstat -cd ~/dev/go/src/k8s.io/kubernetes/ +GITDM_HOME=${GITDM_HOME:-`cd "$(dirname "$0")/.." && pwd`} +K8S_REPO=${K8S_REPO:-$HOME/dev/go/src/k8s.io/kubernetes/} +CNCFDM="$GITDM_HOME/src/cncfdm.py" +cd "$K8S_REPO" git config merge.renameLimit 100000 git config diff.renameLimit 100000 # -m --> map unknowns to 'DomainName *' , -u map unknowns to '(Unknown)' -git log --all -p -M $1..$2 | ~/dev/cncf/gitdm/cncfdm.py -r '^vendor/|/vendor/|^Godeps/' -R -b ~/dev/cncf/gitdm/ -t -z -d -D -U -u -h $FNP.html -o $FNP.txt -x $FNP.csv -git log --all --numstat -M $1..$2 | ~/dev/cncf/gitdm/cncfdm.py -r '^vendor/|/vendor/|^Godeps/' -R -n -b ~/dev/cncf/gitdm/ -t -z -d -D -U -u -h $FNN.html -o $FNN.txt -x $FNN.csv > $FNN.out +git log --all -p -M $1..$2 | "$CNCFDM" -r '^vendor/|/vendor/|^Godeps/' -R -b "$GITDM_HOME/src/" -t -z -d -D -U -u -h $FNP.html -o $FNP.txt -x $FNP.csv +git log --all --numstat -M $1..$2 | "$CNCFDM" -r '^vendor/|/vendor/|^Godeps/' -R -n -b "$GITDM_HOME/src/" -t -z -d -D -U -u -h $FNN.html -o $FNN.txt -x $FNN.csv > $FNN.out git config --unset diff.renameLimit git config --unset merge.renameLimit ls -l $FNP* $FNN* diff --git a/src/run_no_map.sh b/src/run_no_map.sh index 3e9c9720..b235ef6f 100755 --- a/src/run_no_map.sh +++ b/src/run_no_map.sh @@ -2,13 +2,15 @@ PWD=`pwd` FNP=$PWD/run_no_map_patch FNN=$PWD/run_no_map_numstat -cd ~/dev/go/src/k8s.io/kubernetes/ +GITDM_HOME=${GITDM_HOME:-`cd "$(dirname "$0")/.." && pwd`} +K8S_REPO=${K8S_REPO:-$HOME/dev/go/src/k8s.io/kubernetes/} +CNCFDM="$GITDM_HOME/src/cncfdm.py" +cd "$K8S_REPO" git config merge.renameLimit 100000 git config diff.renameLimit 100000 # -m --> map unknowns to 'DomainName *' , -u map unknowns to '(Unknown)' -git log --all -p -M | ~/dev/cncf/gitdm/cncfdm.py -r '^vendor/|/vendor/|^Godeps/' -R -b ~/dev/cncf/gitdm/ -t -z -d -D -U -h $FNP.html -o $FNP.txt -x $FNP.csv -git log --all --numstat -M | ~/dev/cncf/gitdm/cncfdm.py -r '^vendor/|/vendor/|^Godeps/' -R -n -b ~/dev/cncf/gitdm/ -t -z -d -D -U -h $FNN.html -o $FNN.txt -x $FNN.csv > $FNN.out +git log --all -p -M | "$CNCFDM" -r '^vendor/|/vendor/|^Godeps/' -R -b "$GITDM_HOME/src/" -t -z -d -D -U -h $FNP.html -o $FNP.txt -x $FNP.csv +git log --all --numstat -M | "$CNCFDM" -r '^vendor/|/vendor/|^Godeps/' -R -n -b "$GITDM_HOME/src/" -t -z -d -D -U -h $FNN.html -o $FNN.txt -x $FNN.csv > $FNN.out git config --unset diff.renameLimit git config --unset merge.renameLimit cd $PWD - diff --git a/src/run_with_map.sh b/src/run_with_map.sh index 01d97747..99dca798 100755 --- a/src/run_with_map.sh +++ b/src/run_with_map.sh @@ -2,12 +2,15 @@ PWD=`pwd` FNP=$PWD/run_with_map_patch FNN=$PWD/run_with_map_numstat -cd ~/dev/go/src/k8s.io/kubernetes/ +GITDM_HOME=${GITDM_HOME:-`cd "$(dirname "$0")/.." && pwd`} +K8S_REPO=${K8S_REPO:-$HOME/dev/go/src/k8s.io/kubernetes/} +CNCFDM="$GITDM_HOME/src/cncfdm.py" +cd "$K8S_REPO" git config merge.renameLimit 100000 git config diff.renameLimit 100000 # -m --> map unknowns to 'DomainName *' , -u map unknowns to '(Unknown)' -git log --all -p -M | ~/dev/cncf/gitdm/cncfdm.py -r '^vendor/|/vendor/|^Godeps/' -R -b ~/dev/cncf/gitdm/ -t -z -d -D -U -m -h $FNP.html -o $FNP.txt -x $FNP.csv -git log --all --numstat -M | ~/dev/cncf/gitdm/cncfdm.py -r '^vendor/|/vendor/|^Godeps/' -R -n -b ~/dev/cncf/gitdm/ -t -z -d -D -U -m -h $FNN.html -o $FNN.txt -x $FNN.csv > $FNN.out +git log --all -p -M | "$CNCFDM" -r '^vendor/|/vendor/|^Godeps/' -R -b "$GITDM_HOME/src/" -t -z -d -D -U -m -h $FNP.html -o $FNP.txt -x $FNP.csv +git log --all --numstat -M | "$CNCFDM" -r '^vendor/|/vendor/|^Godeps/' -R -n -b "$GITDM_HOME/src/" -t -z -d -D -U -m -h $FNN.html -o $FNN.txt -x $FNN.csv > $FNN.out git config --unset diff.renameLimit git config --unset merge.renameLimit cd $PWD