Skip to content

Fix runner scripts and PowerShell wrapper (correct cncfdm path + stdin passthrough) #947

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
shbhmexe wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Fix runner scripts and PowerShell wrapper (correct cncfdm path + stdin passthrough) #947

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 39 additions & 22 deletions gitdm.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,37 +5,54 @@ $ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
$BaseDir = Join-Path $ScriptDir 'src'
$Engine = Join-Path $BaseDir 'cncfdm.py'

function Test-Python2 {
param(
[Parameter(Mandatory=$true)] [string]$Exe,
[string[]]$ExeArgs = @()
)
try {
& $Exe @ExeArgs -c "import sys; sys.exit(0 if sys.version_info[0] == 2 else 1)" *> $null
return ($LASTEXITCODE -eq 0)
} catch {
return $false
}
}

function Pick-Python {
if ($env:GITDM_PY) { return $env:GITDM_PY }
foreach ($cand in @('py -2','python2','pypy','pypy2')) {
try {
if ($cand -eq 'py -2') {
& py -2 - <<'PY'
import sys
print(2 if sys.version_info[0]==2 else 3)
PY
if ($LASTEXITCODE -eq 0) { return 'py -2' }
} else {
if (Get-Command ($cand.Split(' ')[0]) -ErrorAction SilentlyContinue) { return $cand }
}
} catch { }
# Allow override via environment variable.
if ($env:GITDM_PY) {
return [pscustomobject]@{ Exe = $env:GITDM_PY; Args = @() }
}

# Prefer Python Launcher (if Python 2 is installed).
if (Get-Command py -ErrorAction SilentlyContinue) {
if (Test-Python2 -Exe 'py' -ExeArgs @('-2')) {
return [pscustomobject]@{ Exe = 'py'; Args = @('-2') }
}
}
if (Get-Command python -ErrorAction SilentlyContinue) {
$ver = & python - <<'PY'
import sys
print(sys.version_info[0])
PY
if ($ver -eq 2) { return 'python' }

# Prefer explicit Python 2 binaries, then fall back to python only if it is v2.
foreach ($cand in @('python2', 'pypy2', 'pypy', 'python')) {
if (Get-Command $cand -ErrorAction SilentlyContinue) {
if (Test-Python2 -Exe $cand) {
return [pscustomobject]@{ Exe = $cand; Args = @() }
}
}
}

return $null
}

$py = Pick-Python
if (-not $py) {
Write-Error 'gitdm requires Python 2 or PyPy. Install python2/pypy or set GITDM_PY.'
Write-Error 'gitdm requires Python 2 or PyPy. Install python2/pypy2 or set GITDM_PY.'
exit 1
}

# Pass through stdin/stdout; add -b to point at src/
& $py $Engine -b "$BaseDir/" @Args
exit $LASTEXITCODE
if ($MyInvocation.ExpectingInput) {
$input | & $py.Exe @($py.Args) $Engine -b "$BaseDir/" @Args
} else {
& $py.Exe @($py.Args) $Engine -b "$BaseDir/" @Args
}
exit $LASTEXITCODE
3 changes: 2 additions & 1 deletion src/all_no_map.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
GITDM_HOME=${GITDM_HOME:-`cd "$(dirname "$0")/.." && pwd`}
./run_no_map.sh
mv run_no_map_* ~/dev/cncf/gitdm/kubernetes/all_time/
mv run_no_map_* "$GITDM_HOME/kubernetes/all_time/"
3 changes: 2 additions & 1 deletion src/all_with_map.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
GITDM_HOME=${GITDM_HOME:-`cd "$(dirname "$0")/.." && pwd`}
./run_with_map.sh
mv run_with_map_* ~/dev/cncf/gitdm/kubernetes/all_time/
mv run_with_map_* "$GITDM_HOME/kubernetes/all_time/"
4 changes: 3 additions & 1 deletion src/anyrepo.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
#!/bin/sh
PWD=`pwd`
FN=$PWD/repos/$2
GITDM_HOME=${GITDM_HOME:-`cd "$(dirname "$0")/.." && pwd`}
CNCFDM="$GITDM_HOME/src/cncfdm.py"
cd "$1"
echo "Processing repo $1 $2"
git config merge.renameLimit 100000
git config diff.renameLimit 100000
git log --all --numstat -M | ~/dev/cncf/gitdm/cncfdm.py -r '^vendor/|/vendor/|^Godeps/' -R -n -b ~/dev/cncf/gitdm/ -t -z -d -D -U -u -h $FN.html -o $FN.txt -x $FN.csv > $FN.out
git log --all --numstat -M | "$CNCFDM" -r '^vendor/|/vendor/|^Godeps/' -R -n -b "$GITDM_HOME/src/" -t -z -d -D -U -u -h $FN.html -o $FN.txt -x $FN.csv > $FN.out
git config --unset diff.renameLimit
git config --unset merge.renameLimit
4 changes: 3 additions & 1 deletion src/anyreporange.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
#!/bin/sh
PWD=`pwd`
FN=$PWD/repo_$2_$3
GITDM_HOME=${GITDM_HOME:-`cd "$(dirname "$0")/.." && pwd`}
CNCFDM="$GITDM_HOME/src/cncfdm.py"
cd "$1"
git config merge.renameLimit 100000
git config diff.renameLimit 100000
git log --all --numstat -M --since "$2" --until "$3" | ~/dev/cncf/gitdm/cncfdm.py -r '^vendor/|/vendor/|^Godeps/' -R -n -b ~/dev/cncf/gitdm/ -t -z -d -D -U -u -f "$2" -e "$3" -h $FN.html -o $FN.txt -x $FN.csv > $FN.out
git log --all --numstat -M --since "$2" --until "$3" | "$CNCFDM" -r '^vendor/|/vendor/|^Godeps/' -R -n -b "$GITDM_HOME/src/" -t -z -d -D -U -u -f "$2" -e "$3" -h $FN.html -o $FN.txt -x $FN.csv > $FN.out
git config --unset diff.renameLimit
git config --unset merge.renameLimit
12 changes: 9 additions & 3 deletions src/commits_in_ranges.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,17 @@
#!/bin/sh
if [ $# -lt 4 ]; then
echo "Usage: $0 prefix git.log dt_from dt_to"
exit 1
fi
echo "Args git.log dt_from dt_to"
WD=`pwd`
PREFIX=$1
FN=$2
F=$WD/other_repos/$1_range_unknown_$3_$4
F2=$WD/other_repos/$1_range_no_map_$3_$4
F3=$WD/other_repos/$1_range_with_map_$3_$4
cat $FN | ~/dev/cncf/gitdm/cncfdm.py -f "$3" -e "$4" -r '^vendor/|/vendor/|^Godeps/' -R -n -b ~/dev/cncf/gitdm/ -t -z -d -D -U -u -h $F.html -o $F.txt -x $F.csv > $F.out
#cat $FN | ~/dev/cncf/gitdm/cncfdm.py -f "$3" -e "$4" -r '^vendor/|/vendor/|^Godeps/' -R -n -b ~/dev/cncf/gitdm/ -t -z -d -D -U -h $F2.html -o $F2.txt -x $F2.csv > $F2.out
#cat $FN | ~/dev/cncf/gitdm/cncfdm.py -f "$3" -e "$4" -r '^vendor/|/vendor/|^Godeps/' -R -n -b ~/dev/cncf/gitdm/ -t -z -d -D -U -m -h $F3.html -o $F3.txt -x $F3.csv > $F3.out
GITDM_HOME=${GITDM_HOME:-`cd "$(dirname "$0")/.." && pwd`}
CNCFDM="$GITDM_HOME/src/cncfdm.py"
cat $FN | "$CNCFDM" -f "$3" -e "$4" -r '^vendor/|/vendor/|^Godeps/' -R -n -b "$GITDM_HOME/src/" -t -z -d -D -U -u -h $F.html -o $F.txt -x $F.csv > $F.out
#cat $FN | "$CNCFDM" -f "$3" -e "$4" -r '^vendor/|/vendor/|^Godeps/' -R -n -b "$GITDM_HOME/src/" -t -z -d -D -U -h $F2.html -o $F2.txt -x $F2.csv > $F2.out
#cat $FN | "$CNCFDM" -f "$3" -e "$4" -r '^vendor/|/vendor/|^Godeps/' -R -n -b "$GITDM_HOME/src/" -t -z -d -D -U -m -h $F3.html -o $F3.txt -x $F3.csv > $F3.out
8 changes: 5 additions & 3 deletions src/multirepo.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ FN=$WD/git.log
F=$WD/repos/combined
F2=$WD/repos/combined_no_map
F3=$WD/repos/combined_with_map
GITDM_HOME=${GITDM_HOME:-`cd "$(dirname "$0")/.." && pwd`}
CNCFDM="$GITDM_HOME/src/cncfdm.py"
> $FN
for var in "$@"
do
Expand All @@ -18,9 +20,9 @@ do
done
PWD=$WD
cd $PWD
cat $FN | ~/dev/cncf/gitdm/cncfdm.py -r '^vendor/|/vendor/|^Godeps/' -R -n -b ~/dev/cncf/gitdm/ -t -z -d -D -U -u -h $F.html -o $F.txt -x $F.csv > $F.out
cat $FN | ~/dev/cncf/gitdm/cncfdm.py -r '^vendor/|/vendor/|^Godeps/' -R -n -b ~/dev/cncf/gitdm/ -t -z -d -D -U -h $F2.html -o $F2.txt -x $F2.csv > $F2.out
cat $FN | ~/dev/cncf/gitdm/cncfdm.py -r '^vendor/|/vendor/|^Godeps/' -R -n -b ~/dev/cncf/gitdm/ -t -z -d -D -U -m -h $F3.html -o $F3.txt -x $F3.csv > $F3.out
cat $FN | "$CNCFDM" -r '^vendor/|/vendor/|^Godeps/' -R -n -b "$GITDM_HOME/src/" -t -z -d -D -U -u -h $F.html -o $F.txt -x $F.csv > $F.out
cat $FN | "$CNCFDM" -r '^vendor/|/vendor/|^Godeps/' -R -n -b "$GITDM_HOME/src/" -t -z -d -D -U -h $F2.html -o $F2.txt -x $F2.csv > $F2.out
cat $FN | "$CNCFDM" -r '^vendor/|/vendor/|^Godeps/' -R -n -b "$GITDM_HOME/src/" -t -z -d -D -U -m -h $F3.html -o $F3.txt -x $F3.csv > $F3.out
./commits_in_default_ranges.sh all_kubernetes $FN
#rm -f $FN
#xz -9 $FN
2 changes: 1 addition & 1 deletion src/rerun_data.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ echo "Update PULL_DATE after this, and update datasheet dates and report dates t
echo "Also update last_processed.txt with last processed unknown/not found affliation from repos/combined.txt"
rm -f ./other_repos/*
echo "All with map to (Unknown)"
./all.sh
./run_all.sh
echo "All without mapping"
./all_no_map.sh
echo "All with map to Domain *"
Expand Down
3 changes: 2 additions & 1 deletion src/run_all.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
GITDM_HOME=${GITDM_HOME:-`cd "$(dirname "$0")/.." && pwd`}
./run.sh
mv first_run_* ~/dev/cncf/gitdm/kubernetes/all_time/
mv first_run_* "$GITDM_HOME/kubernetes/all_time/"
11 changes: 7 additions & 4 deletions src/run_for_rels_no_map.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,18 +1,21 @@
#!/bin/sh
if [ $# -lt 2 ]; then
echo "$0 tag1 tag2"
echo "Use "git tag -l" to see available tags"
echo 'Use "git tag -l" to see available tags'
exit 1
fi
PWD=`pwd`
FNP=$PWD/output_no_map_patch
FNN=$PWD/output_no_map_numstat
cd ~/dev/go/src/k8s.io/kubernetes/
GITDM_HOME=${GITDM_HOME:-`cd "$(dirname "$0")/.." && pwd`}
K8S_REPO=${K8S_REPO:-$HOME/dev/go/src/k8s.io/kubernetes/}
CNCFDM="$GITDM_HOME/src/cncfdm.py"
cd "$K8S_REPO"
git config merge.renameLimit 100000
git config diff.renameLimit 100000
# -m --> map unknowns to 'DomainName *' , -u map unknowns to '(Unknown)'
git log --all -p -M $1..$2 | ~/dev/cncf/gitdm/cncfdm.py -r '^vendor/|/vendor/|^Godeps/' -R -b ~/dev/cncf/gitdm/ -t -z -d -D -U -h $FNP.html -o $FNP.txt -x $FNP.csv
git log --all --numstat -M $1..$2 | ~/dev/cncf/gitdm/cncfdm.py -r '^vendor/|/vendor/|^Godeps/' -R -n -b ~/dev/cncf/gitdm/ -t -z -d -D -U -h $FNN.html -o $FNN.txt -x $FNN.csv > $FNN.out
git log --all -p -M $1..$2 | "$CNCFDM" -r '^vendor/|/vendor/|^Godeps/' -R -b "$GITDM_HOME/src/" -t -z -d -D -U -h $FNP.html -o $FNP.txt -x $FNP.csv
git log --all --numstat -M $1..$2 | "$CNCFDM" -r '^vendor/|/vendor/|^Godeps/' -R -n -b "$GITDM_HOME/src/" -t -z -d -D -U -h $FNN.html -o $FNN.txt -x $FNN.csv > $FNN.out
git config --unset diff.renameLimit
git config --unset merge.renameLimit
ls -l $FNP* $FNN*
Expand Down
11 changes: 7 additions & 4 deletions src/run_for_rels_strict.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,18 +1,21 @@
#!/bin/sh
if [ $# -lt 2 ]; then
echo "$0 tag1 tag2"
echo "Use "git tag -l" to see available tags"
echo 'Use "git tag -l" to see available tags'
exit 1
fi
PWD=`pwd`
FNP=$PWD/output_strict_patch
FNN=$PWD/output_strict_numstat
cd ~/dev/go/src/k8s.io/kubernetes/
GITDM_HOME=${GITDM_HOME:-`cd "$(dirname "$0")/.." && pwd`}
K8S_REPO=${K8S_REPO:-$HOME/dev/go/src/k8s.io/kubernetes/}
CNCFDM="$GITDM_HOME/src/cncfdm.py"
cd "$K8S_REPO"
git config merge.renameLimit 100000
git config diff.renameLimit 100000
# -m --> map unknowns to 'DomainName *' , -u map unknowns to '(Unknown)'
git log --all -p -M $1..$2 | ~/dev/cncf/gitdm/cncfdm.py -r '^vendor/|/vendor/|^Godeps/' -R -b ~/dev/cncf/gitdm/ -t -z -d -D -U -u -h $FNP.html -o $FNP.txt -x $FNP.csv
git log --all --numstat -M $1..$2 | ~/dev/cncf/gitdm/cncfdm.py -r '^vendor/|/vendor/|^Godeps/' -R -n -b ~/dev/cncf/gitdm/ -t -z -d -D -U -u -h $FNN.html -o $FNN.txt -x $FNN.csv > $FNN.out
git log --all -p -M $1..$2 | "$CNCFDM" -r '^vendor/|/vendor/|^Godeps/' -R -b "$GITDM_HOME/src/" -t -z -d -D -U -u -h $FNP.html -o $FNP.txt -x $FNP.csv
git log --all --numstat -M $1..$2 | "$CNCFDM" -r '^vendor/|/vendor/|^Godeps/' -R -n -b "$GITDM_HOME/src/" -t -z -d -D -U -u -h $FNN.html -o $FNN.txt -x $FNN.csv > $FNN.out
git config --unset diff.renameLimit
git config --unset merge.renameLimit
ls -l $FNP* $FNN*
Expand Down
10 changes: 6 additions & 4 deletions src/run_no_map.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,15 @@
PWD=`pwd`
FNP=$PWD/run_no_map_patch
FNN=$PWD/run_no_map_numstat
cd ~/dev/go/src/k8s.io/kubernetes/
GITDM_HOME=${GITDM_HOME:-`cd "$(dirname "$0")/.." && pwd`}
K8S_REPO=${K8S_REPO:-$HOME/dev/go/src/k8s.io/kubernetes/}
CNCFDM="$GITDM_HOME/src/cncfdm.py"
cd "$K8S_REPO"
git config merge.renameLimit 100000
git config diff.renameLimit 100000
# -m --> map unknowns to 'DomainName *' , -u map unknowns to '(Unknown)'
git log --all -p -M | ~/dev/cncf/gitdm/cncfdm.py -r '^vendor/|/vendor/|^Godeps/' -R -b ~/dev/cncf/gitdm/ -t -z -d -D -U -h $FNP.html -o $FNP.txt -x $FNP.csv
git log --all --numstat -M | ~/dev/cncf/gitdm/cncfdm.py -r '^vendor/|/vendor/|^Godeps/' -R -n -b ~/dev/cncf/gitdm/ -t -z -d -D -U -h $FNN.html -o $FNN.txt -x $FNN.csv > $FNN.out
git log --all -p -M | "$CNCFDM" -r '^vendor/|/vendor/|^Godeps/' -R -b "$GITDM_HOME/src/" -t -z -d -D -U -h $FNP.html -o $FNP.txt -x $FNP.csv
git log --all --numstat -M | "$CNCFDM" -r '^vendor/|/vendor/|^Godeps/' -R -n -b "$GITDM_HOME/src/" -t -z -d -D -U -h $FNN.html -o $FNN.txt -x $FNN.csv > $FNN.out
git config --unset diff.renameLimit
git config --unset merge.renameLimit
cd $PWD

9 changes: 6 additions & 3 deletions src/run_with_map.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,15 @@
PWD=`pwd`
FNP=$PWD/run_with_map_patch
FNN=$PWD/run_with_map_numstat
cd ~/dev/go/src/k8s.io/kubernetes/
GITDM_HOME=${GITDM_HOME:-`cd "$(dirname "$0")/.." && pwd`}
K8S_REPO=${K8S_REPO:-$HOME/dev/go/src/k8s.io/kubernetes/}
CNCFDM="$GITDM_HOME/src/cncfdm.py"
cd "$K8S_REPO"
git config merge.renameLimit 100000
git config diff.renameLimit 100000
# -m --> map unknowns to 'DomainName *' , -u map unknowns to '(Unknown)'
git log --all -p -M | ~/dev/cncf/gitdm/cncfdm.py -r '^vendor/|/vendor/|^Godeps/' -R -b ~/dev/cncf/gitdm/ -t -z -d -D -U -m -h $FNP.html -o $FNP.txt -x $FNP.csv
git log --all --numstat -M | ~/dev/cncf/gitdm/cncfdm.py -r '^vendor/|/vendor/|^Godeps/' -R -n -b ~/dev/cncf/gitdm/ -t -z -d -D -U -m -h $FNN.html -o $FNN.txt -x $FNN.csv > $FNN.out
git log --all -p -M | "$CNCFDM" -r '^vendor/|/vendor/|^Godeps/' -R -b "$GITDM_HOME/src/" -t -z -d -D -U -m -h $FNP.html -o $FNP.txt -x $FNP.csv
git log --all --numstat -M | "$CNCFDM" -r '^vendor/|/vendor/|^Godeps/' -R -n -b "$GITDM_HOME/src/" -t -z -d -D -U -m -h $FNN.html -o $FNN.txt -x $FNN.csv > $FNN.out
git config --unset diff.renameLimit
git config --unset merge.renameLimit
cd $PWD