Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

sync: add a option to match filter again full path #4492

Merged
merged 1 commit into from
Mar 14, 2024
Merged

sync: add a option to match filter again full path #4492

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions cmd/sync.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,10 @@ func selectionFlags() []cli.Flag {
Name: "include",
Usage: "don't exclude Key matching PATTERN, need to be used with \"--exclude\" option",
},
&cli.BoolFlag{
Name: "match-full-path",
Usage: "match filters again the full path",
},
&cli.Int64Flag{
Name: "limit",
Usage: "limit the number of objects that will be processed (-1 is unlimited, 0 is to process nothing)",
Expand Down
2 changes: 2 additions & 0 deletions pkg/sync/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ type Config struct {
Dry bool
DeleteSrc bool
DeleteDst bool
MatchFullPath bool
Dirs bool
Exclude []string
Include []string
Expand Down Expand Up @@ -147,6 +148,7 @@ func NewConfigFromCli(c *cli.Context) *Config {
DeleteDst: c.Bool("delete-dst"),
Exclude: c.StringSlice("exclude"),
Include: c.StringSlice("include"),
MatchFullPath: c.Bool("match-full-path"),
Existing: c.Bool("existing"),
IgnoreExisting: c.Bool("ignore-existing"),
Links: c.Bool("links"),
Expand Down
48 changes: 39 additions & 9 deletions pkg/sync/sync.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -764,14 +764,14 @@ func startSingleProducer(tasks chan<- object.Object, src, dst object.ObjectStora
return fmt.Errorf("list %s: %s", dst, err)
}

produce(tasks, src, dst, srckeys, dstkeys, config)
produce(tasks, srckeys, dstkeys, config)
return nil
}

func produce(tasks chan<- object.Object, src, dst object.ObjectStorage, srckeys, dstkeys <-chan object.Object, config *Config) {
func produce(tasks chan<- object.Object, srckeys, dstkeys <-chan object.Object, config *Config) {
if len(config.rules) > 0 {
srckeys = filter(srckeys, config.rules)
dstkeys = filter(dstkeys, config.rules)
srckeys = filter(srckeys, config.rules, config)
dstkeys = filter(dstkeys, config.rules, config)
}
var dstobj object.Object
for obj := range srckeys {
Expand Down Expand Up @@ -909,14 +909,20 @@ func parseIncludeRules(args []string) (rules []rule) {
return
}

func filter(keys <-chan object.Object, rules []rule) <-chan object.Object {
func filter(keys <-chan object.Object, rules []rule, config *Config) <-chan object.Object {
r := make(chan object.Object)
go func() {
for o := range keys {
if o == nil {
break
}
if matchKey(rules, o.Key()) {
var ok bool
if config.MatchFullPath {
ok = matchFullPath(rules, o.Key())
} else {
ok = matchLeveledPath(rules, o.Key())
}
if ok {
r <- o
} else {
logger.Debugf("exclude %s", o.Key())
Expand Down Expand Up @@ -1005,8 +1011,32 @@ func matchSuffix(p, s []string) bool {
}
}

func matchFullPath(rules []rule, key string) bool {
ps := strings.Split(key, "/")
for _, rule := range rules {
p := strings.Split(rule.pattern, "/")
var ok bool
if p[0] == "" {
if ps[0] != "" {
p = p[1:]
}
ok = matchPrefix(p, ps)
} else {
ok = matchSuffix(p, ps)
}
if ok {
if rule.include {
break // try next level
} else {
return false
}
}
}
return true
}

// Consistent with rsync behavior, the matching order is adjusted according to the order of the "include" and "exclude" options
func matchKey(rules []rule, key string) bool {
func matchLeveledPath(rules []rule, key string) bool {
parts := strings.Split(key, "/")
for i := range parts {
if parts[i] == "" {
Expand Down Expand Up @@ -1094,7 +1124,7 @@ func startProducer(tasks chan<- object.Object, src, dst object.ObjectStorage, pr
processing[c.Key()] = true
mu.Unlock()

if len(config.rules) > 0 && !matchKey(config.rules, c.Key()) {
if len(config.rules) > 0 && !matchLeveledPath(config.rules, c.Key()) {
logger.Infof("exclude prefix %s", c.Key())
continue
}
Expand Down Expand Up @@ -1144,7 +1174,7 @@ func startProducer(tasks chan<- object.Object, src, dst object.ObjectStorage, pr
return fmt.Errorf("list %s with delimiter: %s", dst, err)
}
// sync returned objects
produce(tasks, src, dst, srckeys, dstkeys, config)
produce(tasks, srckeys, dstkeys, config)
// consume all the keys from dst
for range dstkeys {
}
Expand Down
58 changes: 57 additions & 1 deletion pkg/sync/sync_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -656,12 +656,68 @@ func TestMatchObjects(t *testing.T) {
{rules: []rule{{pattern: "**aa**", include: true}, {pattern: "a"}}, key: "aa/a", want: true},
}
for _, c := range tests {
if got := matchKey(c.rules, c.key); got != c.want {
if got := matchLeveledPath(c.rules, c.key); got != c.want {
t.Errorf("matchKey(%+v, %s) = %v, want %v", c.rules, c.key, got, c.want)
}
}
}

func TestMatchFullPatch(t *testing.T) {
type tcase struct {
rules []rule
key string
}
matchedCases := []tcase{
{rules: []rule{{pattern: "a"}}, key: "b/a"},
{rules: []rule{{pattern: "a*"}}, key: "a1"},
{rules: []rule{{pattern: "a*/b*"}}, key: "a1/b1"},
{rules: []rule{{pattern: "/a*"}}, key: "/a1"},
{rules: []rule{{pattern: "a*/b?/"}}, key: "a1/b1/"},
{rules: []rule{{pattern: "a/**/b"}}, key: "a/c/b"},
{rules: []rule{{pattern: "a/**/b"}}, key: "a/c/d/b"},
{rules: []rule{{pattern: "a/**/b"}}, key: "a/c/d/e/b"},
{rules: []rule{{pattern: "/**/b"}}, key: "a/c/b"},
{rules: []rule{{pattern: "a**/b"}}, key: "a/c/d/b"},
{rules: []rule{{pattern: "a**b"}}, key: "a/c/d/b"},
{rules: []rule{{pattern: "**a"}}, key: "a"},
{rules: []rule{{pattern: "a**"}}, key: "a"},
{rules: []rule{{pattern: "**/d2/**a"}}, key: "/d2/d3/1a"},
{rules: []rule{{pattern: "**/d2/**a"}}, key: "d2/d3/1a"},
}
for _, c := range matchedCases {
if got := matchFullPath(c.rules, c.key); got != false {
t.Errorf("matchKey(%+v, %s) = %v, want %v", c.rules, c.key, got, false)
}
}
unmatchedCases := []tcase{
{rules: []rule{{pattern: "/a"}}, key: "/a1"},
{rules: []rule{{pattern: "a*/b?"}}, key: "a1/b1/c2/d1"},
{rules: []rule{{pattern: "/a/b/c"}}, key: "/a1"},
{rules: []rule{{pattern: "a*/b?/"}}, key: "a1/"},
{rules: []rule{{pattern: "a*/b?/c.txt"}}, key: "a1/b1"},
{rules: []rule{{pattern: "a*/b?/"}}, key: "a1/b1/c.txt"},
{rules: []rule{{pattern: "a*/"}}, key: "a1/b1"},
{rules: []rule{{pattern: "a*/b*/"}}, key: "a1/b1/c1/d.txt/"},
{rules: []rule{{pattern: "/a*/b*"}}, key: "/a1/b1/c1/d.txt/"},
{rules: []rule{{pattern: "a"}}, key: "a/b/c/d/"},
{rules: []rule{{pattern: "a*/b*/c"}}, key: "a1/b1/c1/d.txt/"},
{rules: []rule{{pattern: "a**/b"}}, key: "a/c/d/ab/"},
{rules: []rule{{pattern: "a**b"}}, key: "b/c/d/b"},
{rules: []rule{{pattern: "/**/b"}}, key: "a/c/d/b/"},
{rules: []rule{{pattern: "a?**"}}, key: "a/a"},
{rules: []rule{{pattern: "a**a"}}, key: "a"},
{rules: []rule{{pattern: "aa**a"}}, key: "aa"},
{rules: []rule{{pattern: "a/**/a"}}, key: "a"},
{rules: []rule{{pattern: "a/**/a"}}, key: "a/"},
{rules: []rule{{pattern: "**aa**", include: true}, {pattern: "a"}}, key: "aa/a"},
}
for _, c := range unmatchedCases {
if got := matchFullPath(c.rules, c.key); got != true {
t.Errorf("matchKey(%+v, %s) = %v, want %v", c.rules, c.key, got, true)
}
}
}

func TestParseFilterRule(t *testing.T) {
type tcase struct {
args []string
Expand Down
Loading