From be037bfaf9dff393606474c45253d6943511d072 Mon Sep 17 00:00:00 2001 From: snowfox Date: Thu, 4 Aug 2022 10:49:51 +0800 Subject: [PATCH 1/2] Add --include-regexp and --exclude-regexp parameter, Support regular expressions and path filters. --- lib/const.go | 4 ++++ lib/cp.go | 2 ++ lib/option.go | 6 +++++ lib/sync.go | 2 ++ lib/util.go | 62 ++++++++++++++++++++++++++++++++++++++++++--------- 5 files changed, 65 insertions(+), 11 deletions(-) diff --git a/lib/const.go b/lib/const.go index 32b17062..cabbb067 100644 --- a/lib/const.go +++ b/lib/const.go @@ -44,7 +44,9 @@ const ( OptionDisableCRC64 = "disableCRC64" OptionTimeout = "timeout" OptionInclude = "include" + OptionIncludeRegexp = "includeRegexp" OptionExclude = "exclude" + OptionExcludeRegexp = "excludeRegexp" OptionMeta = "meta" OptionRequestPayer = "payer" OptionLogLevel = "loglevel" @@ -203,7 +205,9 @@ const ( MaxTimeout = MaxInt64 DefaultNonePattern = "" IncludePrompt = "--include" + IncludeRegexpPrompt = "--include-regexp" ExcludePrompt = "--exclude" + ExcludeRegexpPrompt = "--exclude-regexp" MaxAppendObjectSize int64 = 5368709120 MaxBatchCount int = 100 ) diff --git a/lib/cp.go b/lib/cp.go index 55fd8d24..80eef45c 100644 --- a/lib/cp.go +++ b/lib/cp.go @@ -1251,7 +1251,9 @@ var copyCommand = CopyCommand{ OptionRange, OptionEncodingType, OptionInclude, + OptionIncludeRegexp, OptionExclude, + OptionExcludeRegexp, OptionMeta, OptionACL, OptionConfigFile, diff --git a/lib/option.go b/lib/option.go index 886ef111..05fd995f 100644 --- a/lib/option.go +++ b/lib/option.go @@ -93,9 +93,15 @@ var OptionMap = map[string]Option{ OptionInclude: Option{"", "--include", DefaultNonePattern, OptionTypeString, "", "", fmt.Sprintf("包含对象匹配模式，如：*.jpg"), fmt.Sprintf("Include Pattern of key, e.g., *.jpg")}, + OptionIncludeRegexp: Option{"", "--include-regexp", DefaultNonePattern, OptionTypeString, "", "", + fmt.Sprintf("包含对象匹配模式，正则表达式，可包含文件夹，如：\\directory\\"), + fmt.Sprintf("Exclude Directory of key, support regex, support include path, e.g., \\directory\\")}, OptionExclude: Option{"", "--exclude", DefaultNonePattern, OptionTypeString, "", "", fmt.Sprintf("不包含对象匹配模式，如：*.txt"), fmt.Sprintf("Exclude Pattern of key, e.g., *.txt")}, + OptionExcludeRegexp: Option{"", "--exclude-regexp", DefaultNonePattern, OptionTypeString, "", "", + fmt.Sprintf("不包含对象匹配模式，正则表达式，可排除文件夹，如：\\directory\\"), + fmt.Sprintf("Exclude Directory of key, support regex, support exclude path, e.g., \\directory\\")}, OptionMeta: Option{"", "--meta", "", OptionTypeString, "", "", fmt.Sprintf("设置object的meta为[header:value#header:value...]，如：Cache-Control:no-cache#Content-Encoding:gzip"), fmt.Sprintf("Set object meta as [header:value#header:value...], e.g., Cache-Control:no-cache#Content-Encoding:gzip")}, diff --git a/lib/sync.go b/lib/sync.go index 6c1fa1bd..1b8f80dd 100644 --- a/lib/sync.go +++ b/lib/sync.go @@ -384,7 +384,9 @@ var syncCommand = SyncCommand{ OptionRange, OptionEncodingType, OptionInclude, + OptionIncludeRegexp, OptionExclude, + OptionExcludeRegexp, OptionMeta, OptionACL, OptionConfigFile, diff --git a/lib/util.go b/lib/util.go index db7b8a64..ad8c462a 100644 --- a/lib/util.go +++ b/lib/util.go @@ -10,6 +10,7 @@ import ( "os/exec" "os/user" "path/filepath" + "regexp" "runtime" "strconv" "strings" @@ -90,9 +91,9 @@ func getSysInfo() sysInfo { func getUserAgent(ua string) string { sys := getSysInfo() if ua == "" { - return fmt.Sprintf("aliyun-sdk-go/%s (%s/%s/%s;%s)/%s-%s", oss.Version, sys.name, sys.release, sys.machine, runtime.Version(), Package, Version) + return fmt.Sprintf("aliyun-sdk-go/%s (%s/%s/%s;%s)/%s-%s", oss.Version, sys.name, sys.release, sys.machine, runtime.Version(), Package, Version) } - return fmt.Sprintf("aliyun-sdk-go/%s (%s/%s/%s;%s)/%s-%s/%s", oss.Version, sys.name, sys.release, sys.machine, runtime.Version(), Package, Version,ua) + return fmt.Sprintf("aliyun-sdk-go/%s (%s/%s/%s;%s)/%s-%s/%s", oss.Version, sys.name, sys.release, sys.machine, runtime.Version(), Package, Version, ua) } func utcToLocalTime(utc time.Time) time.Time { @@ -192,7 +193,11 @@ func getFilter(cmdline []string) (bool, []filterOptionType) { filters := make([]filterOptionType, 0) for i, item := range cmdline { var strTag = "" - if strings.Index(item, IncludePrompt) == 0 { + if strings.Index(item, IncludeRegexpPrompt) == 0 { + strTag = IncludeRegexpPrompt + } else if strings.Index(item, ExcludeRegexpPrompt) == 0 { + strTag = ExcludeRegexpPrompt + } else if strings.Index(item, IncludePrompt) == 0 { strTag = IncludePrompt } else if strings.Index(item, ExcludePrompt) == 0 { strTag = ExcludePrompt @@ -250,6 +255,16 @@ func filterSingleStr(v, p string, include bool) bool { } } +func filterRegexp(v, p string, include bool) bool { + res, _ := regexp.MatchString(p, v) + + if include { + return res + } else { + return !res + } +} + func filterStrsWithInclude(vs []string, p string) []string { vsf := make([]string, 0) for _, v := range vs { @@ -281,18 +296,43 @@ func matchFiltersForStr(str string, filters []filterOptionType) bool { return true } + var fileNameFilters = make([]filterOptionType, 0) + var regexpFilters = make([]filterOptionType, 0) + for _, filter := range filters { + if strings.EqualFold(filter.name, IncludePrompt) || strings.EqualFold(filter.name, ExcludePrompt) { + fileNameFilters = append(fileNameFilters, filter) + } else if strings.EqualFold(filter.name, IncludeRegexpPrompt) || strings.EqualFold(filter.name, ExcludeRegexpPrompt) { + regexpFilters = append(regexpFilters, filter) + } + } + var res bool - if filters[0].name == IncludePrompt { - res = filterSingleStr(str, filters[0].pattern, true) + + if len(fileNameFilters) > 0 { + if filters[0].name == IncludePrompt { + res = filterSingleStr(str, fileNameFilters[0].pattern, true) + } else { + res = filterSingleStr(str, fileNameFilters[0].pattern, false) + } + + for _, filter := range fileNameFilters[1:] { + if filter.name == IncludePrompt { + res = res || filterSingleStr(str, filter.pattern, true) + } else { + res = res && filterSingleStr(str, filter.pattern, false) + } + } } else { - res = filterSingleStr(str, filters[0].pattern, false) + res = true } - for _, filter := range filters[1:] { - if filter.name == IncludePrompt { - res = res || filterSingleStr(str, filter.pattern, true) - } else { - res = res && filterSingleStr(str, filter.pattern, false) + if len(regexpFilters) > 0 { + for _, filter := range regexpFilters { + if filter.name == IncludeRegexpPrompt { + res = res || filterRegexp(str, filter.pattern, true) + } else { + res = res && filterRegexp(str, filter.pattern, false) + } } } From 6712e7598d28490875f72ee1a0e84621c2070516 Mon Sep 17 00:00:00 2001 From: snowfox Date: Thu, 4 Aug 2022 16:02:40 +0800 Subject: [PATCH 2/2] Fix the problem that the filtered file directory is still created. --- lib/cp.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/cp.go b/lib/cp.go index 80eef45c..a9a91ba5 100644 --- a/lib/cp.go +++ b/lib/cp.go @@ -1904,6 +1904,10 @@ func (cc *CopyCommand) getFileList(dpath string, chFiles chan<- fileInfoType) er } if f.IsDir() { + if !doesSingleFileMatchPatterns(fpath, cc.cpOption.filters) { + return nil + } + if fpath != dpath { if strings.HasSuffix(fileName, "\\") || strings.HasSuffix(fileName, "/") { chFiles <- fileInfoType{fileName, name}