copy-gfs/mktestdata at master · nvsmirnov/copy-gfs · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
#!/bin/sh

set -e

mktest() {
  template="test${1}"
  cnt=""
  while [ -e "$dir/${template}${cnt}.tar" ]; do
    if [ -z "$cnt" ]; then
      cnt=1
    else:
      cnt=$(( $cnt + 1 ))
    fi
  done
  if [ -n "$1" ]; then
    touch -t "$1" "${dir}/${template}${cnt}.tar"
  else
    touch "${dir}/${template}${cnt}.tar"
  fi
}

GFSROOT=data/gfs
mkdir -p "$GFSROOT"

# generate some files that will already be saved in gfs directories befor running copy-gfs - to see which files will be kept and which deleted
for dir in ${GFSROOT}/latest ${GFSROOT}/hourly ${GFSROOT}/minutely ${GFSROOT}/daily ${GFSROOT}/weekly ${GFSROOT}/monthly ${GFSROOT}/yearly; do
  rm -rf "$dir"
  mkdir -p $dir
  touch $dir/file1.tar
  touch $dir/file2.tar
  mktest 201812310100
  mktest 201901310100
  mktest $(date -d '01:00 1 minute ago' '+%Y%m%d%H%M')
  mktest $(date -d '01:00 2 minute ago' '+%Y%m%d%H%M')
  mktest $(date -d '01:00 3 minute ago' '+%Y%m%d%H%M')
  mktest $(date -d '01:00 1 hour ago' '+%Y%m%d%H%M')
  mktest $(date -d '01:00 2 hour ago' '+%Y%m%d%H%M')
  mktest $(date -d '01:00 3 hour ago' '+%Y%m%d%H%M')
  mktest $(date -d '01:00 1 day ago' '+%Y%m%d%H%M')
  mktest $(date -d '01:00 2 day ago' '+%Y%m%d%H%M')
  mktest $(date -d '01:00 3 day ago' '+%Y%m%d%H%M')
  mktest $(date -d '01:00 4 day ago' '+%Y%m%d%H%M')
  mktest $(date -d '01:00 5 day ago' '+%Y%m%d%H%M')
  mktest $(date -d '01:00 6 day ago' '+%Y%m%d%H%M')
  mktest $(date -d '01:00 7 day ago' '+%Y%m%d%H%M')
  mktest $(date -d '01:00 8 day ago' '+%Y%m%d%H%M')
  mktest $(date -d '01:00 9 day ago' '+%Y%m%d%H%M')
  mktest $(date -d '01:00 10 day ago' '+%Y%m%d%H%M')
  mktest $(date -d '01:00 1 month ago' '+%Y%m%d%H%M')
  mktest $(date -d '01:00 10 month ago' '+%Y%m%d%H%M')
  mktest $(date -d '01:00 1 year ago' '+%Y%m%d%H%M')
  mktest $(date -d '01:00 10 year ago' '+%Y%m%d%H%M')
done

KEEPS="--keep-hourly 2 --keep-minutely 2"
mkdir -p data/latest
{
  seq 80  -1 1 | xargs printf '%s months  ago\n'
  seq 375 -5 1 | xargs printf '%s days    ago\n'
  seq 30  -2 1 | xargs printf '%s hours   ago\n'
  seq 80  -2 1 | xargs printf '%s minutes ago\n'
} | while read n; do
  dt=$(date -d "$n" "+%Y%m%d%H%M")
  echo $dt
  touch -t $dt data/latest/test-latest.tar
  # copy file to gfs but do not clean up
  ./copy-gfs $KEEPS -l -s data/latest/test-latest.tar -d ${GFSROOT} -m 'test*.tar'
done
# here, you may look ls -lRtr data/gfs | less and see which files should be kept
# after following command, old files will be deleted
./copy-gfs $KEEPS -l -c -d ${GFSROOT} -m 'test*.tar'
ls -lRtr ${GFSROOT}